From a002475a6a4175cb0fa4ef09e21ef8bcecd84899 Mon Sep 17 00:00:00 2001
From: "Lennart Kats (databricks)" <lennart.kats@databricks.com>
Date: Fri, 27 Dec 2024 12:38:12 +0100
Subject: [PATCH 001/247] Relax checks in builtin template tests (#2042)

## Changes
Relax the checks of `lib/template/builtin_test` so they don't fail for a
local development copy that has uncommitted draft templates. Right now
these tests fail because I have some git-ignored uncommitted templates
in my local dev copy.
---
 libs/template/builtin_test.go | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/libs/template/builtin_test.go b/libs/template/builtin_test.go
index 504e0acca..79e04cb84 100644
--- a/libs/template/builtin_test.go
+++ b/libs/template/builtin_test.go
@@ -11,18 +11,24 @@ import (
 func TestBuiltin(t *testing.T) {
 	out, err := Builtin()
 	require.NoError(t, err)
-	assert.Len(t, out, 3)
+	assert.GreaterOrEqual(t, len(out), 3)
 
-	// Confirm names.
-	assert.Equal(t, "dbt-sql", out[0].Name)
-	assert.Equal(t, "default-python", out[1].Name)
-	assert.Equal(t, "default-sql", out[2].Name)
+	// Create a map of templates by name for easier lookup
+	templates := make(map[string]*BuiltinTemplate)
+	for _, tmpl := range out {
+		templates[tmpl.Name] = &tmpl
+	}
 
-	// Confirm that the filesystems work.
-	_, err = fs.Stat(out[0].FS, `template/{{.project_name}}/dbt_project.yml.tmpl`)
+	// Verify all expected templates exist
+	assert.Contains(t, templates, "dbt-sql")
+	assert.Contains(t, templates, "default-python")
+	assert.Contains(t, templates, "default-sql")
+
+	// Verify the filesystems work for each template
+	_, err = fs.Stat(templates["dbt-sql"].FS, `template/{{.project_name}}/dbt_project.yml.tmpl`)
 	assert.NoError(t, err)
-	_, err = fs.Stat(out[1].FS, `template/{{.project_name}}/tests/main_test.py.tmpl`)
+	_, err = fs.Stat(templates["default-python"].FS, `template/{{.project_name}}/tests/main_test.py.tmpl`)
 	assert.NoError(t, err)
-	_, err = fs.Stat(out[2].FS, `template/{{.project_name}}/src/orders_daily.sql.tmpl`)
+	_, err = fs.Stat(templates["default-sql"].FS, `template/{{.project_name}}/src/orders_daily.sql.tmpl`)
 	assert.NoError(t, err)
 }

From e088d0d996979ddd56dd9ce10d3d1becbdc771f6 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 30 Dec 2024 16:18:57 +0100
Subject: [PATCH 002/247] Add lint.sh to run golanci-lint in 2 stages (#2051)

First stage is to run goimports and formatter, second is full suite.

This ensures that imports and formatting are fixed even in presence of
other issues. Otherwise golanci-lint refuses to fix anything
https://github.com/golangci/golangci-lint/issues/5257

This helpful when running aider with config like this - aider will use
that to autofix what it can after every update:

```
% cat .aider.conf.yml
lint-cmd:
  - "go: ./lint.sh"
```
---
 Makefile | 2 +-
 lint.sh  | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)
 create mode 100755 lint.sh

diff --git a/Makefile b/Makefile
index f8e7834a5..7dca3b2cf 100644
--- a/Makefile
+++ b/Makefile
@@ -2,7 +2,7 @@ default: build
 
 lint: vendor
 	@echo "✓ Linting source code with https://golangci-lint.run/ (with --fix)..."
-	@golangci-lint run --fix ./...
+	@./lint.sh ./...
 
 lintcheck: vendor
 	@echo "✓ Linting source code with https://golangci-lint.run/ ..."
diff --git a/lint.sh b/lint.sh
new file mode 100755
index 000000000..c93d04c66
--- /dev/null
+++ b/lint.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -euo pipefail
+# With golangci-lint, if there are any compliation issues, then formatters' autofix won't be applied.
+# https://github.com/golangci/golangci-lint/issues/5257
+# However, running goimports first alone will actually fix some of the compilation issues.
+# Fixing formatting is also reasonable thing to do.
+# For this reason, this script runs golangci-lint in two stages:
+golangci-lint run --fix --no-config --disable-all --enable gofumpt,goimports $@
+exec golangci-lint run --fix $@

From 261b7f4083c4f10c0cc0e6c0ec9d95bfc2927174 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 30 Dec 2024 16:26:21 +0100
Subject: [PATCH 003/247] Move bulk of "golden tests" logic to libs/testdiff
 (#2054)

## Changes
- Detach "golden files" assertions from testcli runner. Now any output
can be compared, no matter how it is obtained.
- Move those assertion to libs/testdiff package.

This allows using "golden files" in non-integration tests.

## Tests
Existing tests
---
 .../bundle/init_default_python_test.go        |   7 +-
 internal/testcli/golden.go                    | 201 +----------------
 libs/testdiff/golden.go                       | 212 ++++++++++++++++++
 .../testcli => libs/testdiff}/golden_test.go  |   2 +-
 4 files changed, 220 insertions(+), 202 deletions(-)
 create mode 100644 libs/testdiff/golden.go
 rename {internal/testcli => libs/testdiff}/golden_test.go (92%)

diff --git a/integration/bundle/init_default_python_test.go b/integration/bundle/init_default_python_test.go
index 9b65636e9..c93e6b50b 100644
--- a/integration/bundle/init_default_python_test.go
+++ b/integration/bundle/init_default_python_test.go
@@ -11,6 +11,7 @@ import (
 	"github.com/databricks/cli/internal/testcli"
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/python/pythontest"
+	"github.com/databricks/cli/libs/testdiff"
 	"github.com/stretchr/testify/require"
 )
 
@@ -50,14 +51,14 @@ func testDefaultPython(t *testing.T, pythonVersion string) {
 	ctx, wt := acc.WorkspaceTest(t)
 
 	uniqueProjectId := testutil.RandomName("")
-	ctx, replacements := testcli.WithReplacementsMap(ctx)
+	ctx, replacements := testdiff.WithReplacementsMap(ctx)
 	replacements.Set(uniqueProjectId, "$UNIQUE_PRJ")
 
 	user, err := wt.W.CurrentUser.Me(ctx)
 	require.NoError(t, err)
 	require.NotNil(t, user)
-	testcli.PrepareReplacementsUser(t, replacements, *user)
-	testcli.PrepareReplacements(t, replacements, wt.W)
+	testdiff.PrepareReplacementsUser(t, replacements, *user)
+	testdiff.PrepareReplacements(t, replacements, wt.W)
 
 	tmpDir := t.TempDir()
 	testutil.Chdir(t, tmpDir)
diff --git a/internal/testcli/golden.go b/internal/testcli/golden.go
index 34f38f18a..a16f42c72 100644
--- a/internal/testcli/golden.go
+++ b/internal/testcli/golden.go
@@ -3,222 +3,27 @@ package testcli
 import (
 	"context"
 	"fmt"
-	"os"
-	"regexp"
-	"slices"
 	"strings"
-	"testing"
 
 	"github.com/databricks/cli/internal/testutil"
-	"github.com/databricks/cli/libs/iamutil"
 	"github.com/databricks/cli/libs/testdiff"
-	"github.com/databricks/databricks-sdk-go"
-	"github.com/databricks/databricks-sdk-go/service/iam"
 	"github.com/stretchr/testify/assert"
 )
 
-var OverwriteMode = os.Getenv("TESTS_OUTPUT") == "OVERWRITE"
-
-func ReadFile(t testutil.TestingT, ctx context.Context, filename string) string {
-	data, err := os.ReadFile(filename)
-	if os.IsNotExist(err) {
-		return ""
-	}
-	assert.NoError(t, err)
-	// On CI, on Windows \n in the file somehow end up as \r\n
-	return NormalizeNewlines(string(data))
-}
-
 func captureOutput(t testutil.TestingT, ctx context.Context, args []string) string {
 	t.Logf("run args: [%s]", strings.Join(args, ", "))
 	r := NewRunner(t, ctx, args...)
 	stdout, stderr, err := r.Run()
 	assert.NoError(t, err)
-	out := stderr.String() + stdout.String()
-	return ReplaceOutput(t, ctx, out)
-}
-
-func WriteFile(t testutil.TestingT, filename, data string) {
-	t.Logf("Overwriting %s", filename)
-	err := os.WriteFile(filename, []byte(data), 0o644)
-	assert.NoError(t, err)
+	return stderr.String() + stdout.String()
 }
 
 func AssertOutput(t testutil.TestingT, ctx context.Context, args []string, expectedPath string) {
-	expected := ReadFile(t, ctx, expectedPath)
-
 	out := captureOutput(t, ctx, args)
-
-	if out != expected {
-		actual := fmt.Sprintf("Output from %v", args)
-		testdiff.AssertEqualTexts(t, expectedPath, actual, expected, out)
-
-		if OverwriteMode {
-			WriteFile(t, expectedPath, out)
-		}
-	}
+	testdiff.AssertOutput(t, ctx, out, fmt.Sprintf("Output from %v", args), expectedPath)
 }
 
 func AssertOutputJQ(t testutil.TestingT, ctx context.Context, args []string, expectedPath string, ignorePaths []string) {
-	expected := ReadFile(t, ctx, expectedPath)
-
 	out := captureOutput(t, ctx, args)
-
-	if out != expected {
-		actual := fmt.Sprintf("Output from %v", args)
-		testdiff.AssertEqualJQ(t.(*testing.T), expectedPath, actual, expected, out, ignorePaths)
-
-		if OverwriteMode {
-			WriteFile(t, expectedPath, out)
-		}
-	}
-}
-
-var (
-	uuidRegex        = regexp.MustCompile(`[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}`)
-	numIdRegex       = regexp.MustCompile(`[0-9]{3,}`)
-	privatePathRegex = regexp.MustCompile(`(/tmp|/private)(/.*)/([a-zA-Z0-9]+)`)
-)
-
-func ReplaceOutput(t testutil.TestingT, ctx context.Context, out string) string {
-	out = NormalizeNewlines(out)
-	replacements := GetReplacementsMap(ctx)
-	if replacements == nil {
-		t.Fatal("WithReplacementsMap was not called")
-	}
-	out = replacements.Replace(out)
-	out = uuidRegex.ReplaceAllString(out, "<UUID>")
-	out = numIdRegex.ReplaceAllString(out, "<NUMID>")
-	out = privatePathRegex.ReplaceAllString(out, "/tmp/.../$3")
-
-	return out
-}
-
-type key int
-
-const (
-	replacementsMapKey = key(1)
-)
-
-type Replacement struct {
-	Old string
-	New string
-}
-
-type ReplacementsContext struct {
-	Repls []Replacement
-}
-
-func (r *ReplacementsContext) Replace(s string) string {
-	// QQQ Should probably only replace whole words
-	for _, repl := range r.Repls {
-		s = strings.ReplaceAll(s, repl.Old, repl.New)
-	}
-	return s
-}
-
-func (r *ReplacementsContext) Set(old, new string) {
-	if old == "" || new == "" {
-		return
-	}
-	r.Repls = append(r.Repls, Replacement{Old: old, New: new})
-}
-
-func WithReplacementsMap(ctx context.Context) (context.Context, *ReplacementsContext) {
-	value := ctx.Value(replacementsMapKey)
-	if value != nil {
-		if existingMap, ok := value.(*ReplacementsContext); ok {
-			return ctx, existingMap
-		}
-	}
-
-	newMap := &ReplacementsContext{}
-	ctx = context.WithValue(ctx, replacementsMapKey, newMap)
-	return ctx, newMap
-}
-
-func GetReplacementsMap(ctx context.Context) *ReplacementsContext {
-	value := ctx.Value(replacementsMapKey)
-	if value != nil {
-		if existingMap, ok := value.(*ReplacementsContext); ok {
-			return existingMap
-		}
-	}
-	return nil
-}
-
-func PrepareReplacements(t testutil.TestingT, r *ReplacementsContext, w *databricks.WorkspaceClient) {
-	// in some clouds (gcp) w.Config.Host includes "https://" prefix in others it's really just a host (azure)
-	host := strings.TrimPrefix(strings.TrimPrefix(w.Config.Host, "http://"), "https://")
-	r.Set(host, "$DATABRICKS_HOST")
-	r.Set(w.Config.ClusterID, "$DATABRICKS_CLUSTER_ID")
-	r.Set(w.Config.WarehouseID, "$DATABRICKS_WAREHOUSE_ID")
-	r.Set(w.Config.ServerlessComputeID, "$DATABRICKS_SERVERLESS_COMPUTE_ID")
-	r.Set(w.Config.MetadataServiceURL, "$DATABRICKS_METADATA_SERVICE_URL")
-	r.Set(w.Config.AccountID, "$DATABRICKS_ACCOUNT_ID")
-	r.Set(w.Config.Token, "$DATABRICKS_TOKEN")
-	r.Set(w.Config.Username, "$DATABRICKS_USERNAME")
-	r.Set(w.Config.Password, "$DATABRICKS_PASSWORD")
-	r.Set(w.Config.Profile, "$DATABRICKS_CONFIG_PROFILE")
-	r.Set(w.Config.ConfigFile, "$DATABRICKS_CONFIG_FILE")
-	r.Set(w.Config.GoogleServiceAccount, "$DATABRICKS_GOOGLE_SERVICE_ACCOUNT")
-	r.Set(w.Config.GoogleCredentials, "$GOOGLE_CREDENTIALS")
-	r.Set(w.Config.AzureResourceID, "$DATABRICKS_AZURE_RESOURCE_ID")
-	r.Set(w.Config.AzureClientSecret, "$ARM_CLIENT_SECRET")
-	// r.Set(w.Config.AzureClientID, "$ARM_CLIENT_ID")
-	r.Set(w.Config.AzureClientID, "$USERNAME")
-	r.Set(w.Config.AzureTenantID, "$ARM_TENANT_ID")
-	r.Set(w.Config.ActionsIDTokenRequestURL, "$ACTIONS_ID_TOKEN_REQUEST_URL")
-	r.Set(w.Config.ActionsIDTokenRequestToken, "$ACTIONS_ID_TOKEN_REQUEST_TOKEN")
-	r.Set(w.Config.AzureEnvironment, "$ARM_ENVIRONMENT")
-	r.Set(w.Config.ClientID, "$DATABRICKS_CLIENT_ID")
-	r.Set(w.Config.ClientSecret, "$DATABRICKS_CLIENT_SECRET")
-	r.Set(w.Config.DatabricksCliPath, "$DATABRICKS_CLI_PATH")
-	// This is set to words like "path" that happen too frequently
-	// r.Set(w.Config.AuthType, "$DATABRICKS_AUTH_TYPE")
-}
-
-func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam.User) {
-	// There could be exact matches or overlap between different name fields, so sort them by length
-	// to ensure we match the largest one first and map them all to the same token
-	names := []string{
-		u.DisplayName,
-		u.UserName,
-		iamutil.GetShortUserName(&u),
-		u.Name.FamilyName,
-		u.Name.GivenName,
-	}
-	if u.Name != nil {
-		names = append(names, u.Name.FamilyName)
-		names = append(names, u.Name.GivenName)
-	}
-	for _, val := range u.Emails {
-		names = append(names, val.Value)
-	}
-	stableSortReverseLength(names)
-
-	for _, name := range names {
-		r.Set(name, "$USERNAME")
-	}
-
-	for ind, val := range u.Groups {
-		r.Set(val.Value, fmt.Sprintf("$USER.Groups[%d]", ind))
-	}
-
-	r.Set(u.Id, "$USER.Id")
-
-	for ind, val := range u.Roles {
-		r.Set(val.Value, fmt.Sprintf("$USER.Roles[%d]", ind))
-	}
-}
-
-func stableSortReverseLength(strs []string) {
-	slices.SortStableFunc(strs, func(a, b string) int {
-		return len(b) - len(a)
-	})
-}
-
-func NormalizeNewlines(input string) string {
-	output := strings.ReplaceAll(input, "\r\n", "\n")
-	return strings.ReplaceAll(output, "\r", "\n")
+	testdiff.AssertOutputJQ(t, ctx, out, fmt.Sprintf("Output from %v", args), expectedPath, ignorePaths)
 }
diff --git a/libs/testdiff/golden.go b/libs/testdiff/golden.go
new file mode 100644
index 000000000..c338ac9e5
--- /dev/null
+++ b/libs/testdiff/golden.go
@@ -0,0 +1,212 @@
+package testdiff
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"regexp"
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/databricks/cli/internal/testutil"
+	"github.com/databricks/cli/libs/iamutil"
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/service/iam"
+	"github.com/stretchr/testify/assert"
+)
+
+var OverwriteMode = os.Getenv("TESTS_OUTPUT") == "OVERWRITE"
+
+func ReadFile(t testutil.TestingT, ctx context.Context, filename string) string {
+	data, err := os.ReadFile(filename)
+	if os.IsNotExist(err) {
+		return ""
+	}
+	assert.NoError(t, err)
+	// On CI, on Windows \n in the file somehow end up as \r\n
+	return NormalizeNewlines(string(data))
+}
+
+func WriteFile(t testutil.TestingT, filename, data string) {
+	t.Logf("Overwriting %s", filename)
+	err := os.WriteFile(filename, []byte(data), 0o644)
+	assert.NoError(t, err)
+}
+
+func AssertOutput(t testutil.TestingT, ctx context.Context, out, outTitle, expectedPath string) {
+	expected := ReadFile(t, ctx, expectedPath)
+
+	out = ReplaceOutput(t, ctx, out)
+
+	if out != expected {
+		AssertEqualTexts(t, expectedPath, outTitle, expected, out)
+
+		if OverwriteMode {
+			WriteFile(t, expectedPath, out)
+		}
+	}
+}
+
+func AssertOutputJQ(t testutil.TestingT, ctx context.Context, out, outTitle, expectedPath string, ignorePaths []string) {
+	expected := ReadFile(t, ctx, expectedPath)
+
+	out = ReplaceOutput(t, ctx, out)
+
+	if out != expected {
+		AssertEqualJQ(t.(*testing.T), expectedPath, outTitle, expected, out, ignorePaths)
+
+		if OverwriteMode {
+			WriteFile(t, expectedPath, out)
+		}
+	}
+}
+
+var (
+	uuidRegex        = regexp.MustCompile(`[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}`)
+	numIdRegex       = regexp.MustCompile(`[0-9]{3,}`)
+	privatePathRegex = regexp.MustCompile(`(/tmp|/private)(/.*)/([a-zA-Z0-9]+)`)
+)
+
+func ReplaceOutput(t testutil.TestingT, ctx context.Context, out string) string {
+	out = NormalizeNewlines(out)
+	replacements := GetReplacementsMap(ctx)
+	if replacements == nil {
+		t.Fatal("WithReplacementsMap was not called")
+	}
+	out = replacements.Replace(out)
+	out = uuidRegex.ReplaceAllString(out, "<UUID>")
+	out = numIdRegex.ReplaceAllString(out, "<NUMID>")
+	out = privatePathRegex.ReplaceAllString(out, "/tmp/.../$3")
+
+	return out
+}
+
+type key int
+
+const (
+	replacementsMapKey = key(1)
+)
+
+type Replacement struct {
+	Old string
+	New string
+}
+
+type ReplacementsContext struct {
+	Repls []Replacement
+}
+
+func (r *ReplacementsContext) Replace(s string) string {
+	// QQQ Should probably only replace whole words
+	for _, repl := range r.Repls {
+		s = strings.ReplaceAll(s, repl.Old, repl.New)
+	}
+	return s
+}
+
+func (r *ReplacementsContext) Set(old, new string) {
+	if old == "" || new == "" {
+		return
+	}
+	r.Repls = append(r.Repls, Replacement{Old: old, New: new})
+}
+
+func WithReplacementsMap(ctx context.Context) (context.Context, *ReplacementsContext) {
+	value := ctx.Value(replacementsMapKey)
+	if value != nil {
+		if existingMap, ok := value.(*ReplacementsContext); ok {
+			return ctx, existingMap
+		}
+	}
+
+	newMap := &ReplacementsContext{}
+	ctx = context.WithValue(ctx, replacementsMapKey, newMap)
+	return ctx, newMap
+}
+
+func GetReplacementsMap(ctx context.Context) *ReplacementsContext {
+	value := ctx.Value(replacementsMapKey)
+	if value != nil {
+		if existingMap, ok := value.(*ReplacementsContext); ok {
+			return existingMap
+		}
+	}
+	return nil
+}
+
+func PrepareReplacements(t testutil.TestingT, r *ReplacementsContext, w *databricks.WorkspaceClient) {
+	// in some clouds (gcp) w.Config.Host includes "https://" prefix in others it's really just a host (azure)
+	host := strings.TrimPrefix(strings.TrimPrefix(w.Config.Host, "http://"), "https://")
+	r.Set(host, "$DATABRICKS_HOST")
+	r.Set(w.Config.ClusterID, "$DATABRICKS_CLUSTER_ID")
+	r.Set(w.Config.WarehouseID, "$DATABRICKS_WAREHOUSE_ID")
+	r.Set(w.Config.ServerlessComputeID, "$DATABRICKS_SERVERLESS_COMPUTE_ID")
+	r.Set(w.Config.MetadataServiceURL, "$DATABRICKS_METADATA_SERVICE_URL")
+	r.Set(w.Config.AccountID, "$DATABRICKS_ACCOUNT_ID")
+	r.Set(w.Config.Token, "$DATABRICKS_TOKEN")
+	r.Set(w.Config.Username, "$DATABRICKS_USERNAME")
+	r.Set(w.Config.Password, "$DATABRICKS_PASSWORD")
+	r.Set(w.Config.Profile, "$DATABRICKS_CONFIG_PROFILE")
+	r.Set(w.Config.ConfigFile, "$DATABRICKS_CONFIG_FILE")
+	r.Set(w.Config.GoogleServiceAccount, "$DATABRICKS_GOOGLE_SERVICE_ACCOUNT")
+	r.Set(w.Config.GoogleCredentials, "$GOOGLE_CREDENTIALS")
+	r.Set(w.Config.AzureResourceID, "$DATABRICKS_AZURE_RESOURCE_ID")
+	r.Set(w.Config.AzureClientSecret, "$ARM_CLIENT_SECRET")
+	// r.Set(w.Config.AzureClientID, "$ARM_CLIENT_ID")
+	r.Set(w.Config.AzureClientID, "$USERNAME")
+	r.Set(w.Config.AzureTenantID, "$ARM_TENANT_ID")
+	r.Set(w.Config.ActionsIDTokenRequestURL, "$ACTIONS_ID_TOKEN_REQUEST_URL")
+	r.Set(w.Config.ActionsIDTokenRequestToken, "$ACTIONS_ID_TOKEN_REQUEST_TOKEN")
+	r.Set(w.Config.AzureEnvironment, "$ARM_ENVIRONMENT")
+	r.Set(w.Config.ClientID, "$DATABRICKS_CLIENT_ID")
+	r.Set(w.Config.ClientSecret, "$DATABRICKS_CLIENT_SECRET")
+	r.Set(w.Config.DatabricksCliPath, "$DATABRICKS_CLI_PATH")
+	// This is set to words like "path" that happen too frequently
+	// r.Set(w.Config.AuthType, "$DATABRICKS_AUTH_TYPE")
+}
+
+func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam.User) {
+	// There could be exact matches or overlap between different name fields, so sort them by length
+	// to ensure we match the largest one first and map them all to the same token
+	names := []string{
+		u.DisplayName,
+		u.UserName,
+		iamutil.GetShortUserName(&u),
+		u.Name.FamilyName,
+		u.Name.GivenName,
+	}
+	if u.Name != nil {
+		names = append(names, u.Name.FamilyName)
+		names = append(names, u.Name.GivenName)
+	}
+	for _, val := range u.Emails {
+		names = append(names, val.Value)
+	}
+	stableSortReverseLength(names)
+
+	for _, name := range names {
+		r.Set(name, "$USERNAME")
+	}
+
+	for ind, val := range u.Groups {
+		r.Set(val.Value, fmt.Sprintf("$USER.Groups[%d]", ind))
+	}
+
+	r.Set(u.Id, "$USER.Id")
+
+	for ind, val := range u.Roles {
+		r.Set(val.Value, fmt.Sprintf("$USER.Roles[%d]", ind))
+	}
+}
+
+func stableSortReverseLength(strs []string) {
+	slices.SortStableFunc(strs, func(a, b string) int {
+		return len(b) - len(a)
+	})
+}
+
+func NormalizeNewlines(input string) string {
+	output := strings.ReplaceAll(input, "\r\n", "\n")
+	return strings.ReplaceAll(output, "\r", "\n")
+}
diff --git a/internal/testcli/golden_test.go b/libs/testdiff/golden_test.go
similarity index 92%
rename from internal/testcli/golden_test.go
rename to libs/testdiff/golden_test.go
index 215bf33d3..0fc32be21 100644
--- a/internal/testcli/golden_test.go
+++ b/libs/testdiff/golden_test.go
@@ -1,4 +1,4 @@
-package testcli
+package testdiff
 
 import (
 	"testing"

From 1306e5ec6745b36acd7c17c9b3b4aea1f55d6807 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 30 Dec 2024 18:41:45 +0100
Subject: [PATCH 004/247] Add CODEOWNERS (#2055)

Goal is to have DABs core team automatically added as reviewers so that
you don't have to click manually.

Based on this example:
https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#example-of-a-codeowners-file
---
 .github/CODEOWNERS | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 .github/CODEOWNERS

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
new file mode 100644
index 000000000..76835de7d
--- /dev/null
+++ b/.github/CODEOWNERS
@@ -0,0 +1 @@
+* @pietern @andrewnester @shreyas-goenka @denik

From 1ce20a2612031a3d5c7b663203296957671e4f1d Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 30 Dec 2024 19:39:33 +0100
Subject: [PATCH 005/247] lint.sh: read config for formatters; include gofmt
 (#2056)

As suggested here:
https://github.com/databricks/cli/pull/2051#discussion_r1899641273
---
 lint.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lint.sh b/lint.sh
index c93d04c66..13efa9855 100755
--- a/lint.sh
+++ b/lint.sh
@@ -5,5 +5,5 @@ set -euo pipefail
 # However, running goimports first alone will actually fix some of the compilation issues.
 # Fixing formatting is also reasonable thing to do.
 # For this reason, this script runs golangci-lint in two stages:
-golangci-lint run --fix --no-config --disable-all --enable gofumpt,goimports $@
+golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix $@
 exec golangci-lint run --fix $@

From 511db52bb7a47270dd883bb490a8844f9b25fe76 Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Tue, 31 Dec 2024 15:21:13 +0530
Subject: [PATCH 006/247] Remove unnecessary GET call in pipeline runner
 (#1850)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Changes
This GET API call is unnecessary and serves no purpose. Let's remove it.
Noticed this when I was adding a unit test for the pipeline runner here:
https://github.com/databricks/cli/pull/1849

## Tests
Manually.

### Case 1: The pipeline does not exist

Before:
```
➜  my_project git:(master) ✗ databricks bundle run my_project_pipeline -p dogfood
Error: User shreyas.goenka@databricks.com does not have View permissions on pipeline 9941901a-e48b-4d04-b6ba-e0072ad126bg.
```

After:
```
➜  my_project git:(master) ✗ cli bundle run my_project_pipeline -p dogfood
Error: User shreyas.goenka@databricks.com does not have Run permissions on pipeline 9941901a-e48b-4d04-b6ba-e0072ad126bg.
```

### Case 2: Pipeline exists

Before:
```
➜  my_project git:(master) ✗ databricks bundle run my_project_pipeline -p dogfood --restart
Update URL: https://e2-dogfood.staging.cloud.databricks.com/#joblist/pipelines/9941901a-e48b-4d04-b6ba-e0072ad126bf/updates/0f988d62-9ec7-49f1-b429-5572ece3a9aa

2024-11-18T15:30:36.054Z update_progress INFO "Update 0f988d is WAITING_FOR_RESOURCES."
```

After:
```
➜  my_project git:(master) ✗ cli bundle run my_project_pipeline -p dogfood --restart
Update URL: https://e2-dogfood.staging.cloud.databricks.com/#joblist/pipelines/9941901a-e48b-4d04-b6ba-e0072ad126bf/updates/87b43350-6186-4a9b-9d0e-38da2ecf33ae

2024-11-18T15:28:27.144Z update_progress INFO "Update 87b433 is WAITING_FOR_RESOURCES."
```
---
 bundle/run/pipeline.go      | 5 -----
 bundle/run/pipeline_test.go | 2 --
 2 files changed, 7 deletions(-)

diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go
index a0e7d1e1e..d84015d76 100644
--- a/bundle/run/pipeline.go
+++ b/bundle/run/pipeline.go
@@ -90,11 +90,6 @@ func (r *pipelineRunner) Run(ctx context.Context, opts *Options) (output.RunOutp
 	// Include resource key in logger.
 	ctx = log.NewContext(ctx, log.GetLogger(ctx).With("resource", r.Key()))
 	w := r.bundle.WorkspaceClient()
-	_, err := w.Pipelines.GetByPipelineId(ctx, pipelineID)
-	if err != nil {
-		log.Warnf(ctx, "Cannot get pipeline: %s", err)
-		return nil, err
-	}
 
 	req, err := opts.Pipeline.toPayload(r.pipeline, pipelineID)
 	if err != nil {
diff --git a/bundle/run/pipeline_test.go b/bundle/run/pipeline_test.go
index 66f9d86be..bfa0c5846 100644
--- a/bundle/run/pipeline_test.go
+++ b/bundle/run/pipeline_test.go
@@ -90,8 +90,6 @@ func TestPipelineRunnerRestart(t *testing.T) {
 		PipelineId: "123",
 	}).Return(mockWait, nil)
 
-	pipelineApi.EXPECT().GetByPipelineId(mock.Anything, "123").Return(&pipelines.GetPipelineResponse{}, nil)
-
 	// Mock runner starting a new update
 	pipelineApi.EXPECT().StartUpdate(mock.Anything, pipelines.StartUpdate{
 		PipelineId: "123",

From 3f523b45cc59aa2cb4fbbffdf267e8ea094b3ad6 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 31 Dec 2024 15:01:45 +0100
Subject: [PATCH 007/247] Fix lost diags across different mutators (#2057)

## Changes
Fix cases where accumulated diagnostics are lost instead of being
propagated further. In some cases it's not possible, add a comment
there.

## Tests
Existing tests
---
 bundle/artifacts/expand_globs.go                     | 2 +-
 bundle/config/mutator/resolve_resource_references.go | 1 +
 bundle/config/root.go                                | 3 ++-
 bundle/config/validate/folder_permissions.go         | 3 ++-
 4 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/bundle/artifacts/expand_globs.go b/bundle/artifacts/expand_globs.go
index 7d44db0be..c0af7c69e 100644
--- a/bundle/artifacts/expand_globs.go
+++ b/bundle/artifacts/expand_globs.go
@@ -97,7 +97,7 @@ func (m *expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost
 		return dyn.SetByPath(v, base, dyn.V(output))
 	})
 	if err != nil {
-		return diag.FromErr(err)
+		diags = diags.Extend(diag.FromErr(err))
 	}
 
 	return diags
diff --git a/bundle/config/mutator/resolve_resource_references.go b/bundle/config/mutator/resolve_resource_references.go
index bf902f928..20a5b6585 100644
--- a/bundle/config/mutator/resolve_resource_references.go
+++ b/bundle/config/mutator/resolve_resource_references.go
@@ -40,6 +40,7 @@ func (m *resolveResourceReferences) Apply(ctx context.Context, b *bundle.Bundle)
 		})
 	}
 
+	// Note, diags are lost from all goroutines except the first one to return diag
 	return diag.FromErr(errs.Wait())
 }
 
diff --git a/bundle/config/root.go b/bundle/config/root.go
index 4b1467456..91c15fd9d 100644
--- a/bundle/config/root.go
+++ b/bundle/config/root.go
@@ -102,7 +102,8 @@ func LoadFromBytes(path string, raw []byte) (*Root, diag.Diagnostics) {
 	// Convert normalized configuration tree to typed configuration.
 	err = r.updateWithDynamicValue(v)
 	if err != nil {
-		return nil, diag.Errorf("failed to load %s: %v", path, err)
+		diags = diags.Extend(diag.Errorf("failed to load %s: %v", path, err))
+		return nil, diags
 	}
 	return &r, diags
 }
diff --git a/bundle/config/validate/folder_permissions.go b/bundle/config/validate/folder_permissions.go
index aa89a0551..5b28d599e 100644
--- a/bundle/config/validate/folder_permissions.go
+++ b/bundle/config/validate/folder_permissions.go
@@ -36,7 +36,8 @@ func (f *folderPermissions) Apply(ctx context.Context, b bundle.ReadOnlyBundle)
 	}
 
 	if err := g.Wait(); err != nil {
-		return diag.FromErr(err)
+		// Note, only diag from first coroutine is captured, others are lost
+		diags = diags.Extend(diag.FromErr(err))
 	}
 
 	for _, r := range results {

From 3f75240a562d88a9411cdd50a2ba24139ca094b7 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 2 Jan 2025 10:49:21 +0100
Subject: [PATCH 008/247] Improve test output to include correct location
 (#2058)

## Changes
- Add t.Helper() in testcli-related helpers, this ensures that output is
attributed correctly to test case and not to the helper.
- Modify testlcli.Run() to run process in foreground. This is needed for
t.Helper to work.
- Extend a few assertions with message to help attribute it to proper
helper where needed.

## Tests
Manually reviewed test output.

Before:

```
+ go test --timeout 3h -v -run TestDefaultPython/3.9 ./integration/bundle/
=== RUN   TestDefaultPython
=== RUN   TestDefaultPython/3.9
    workspace.go:26: aws
    golden.go:14: run args: [bundle, init, default-python, --config-file, config.json]
    runner.go:206: [databricks stderr]:
    runner.go:206: [databricks stderr]: Welcome to the default Python template for Databricks Asset Bundles!
...
    testdiff.go:23:
                Error Trace:    /Users/denis.bilenko/work/cli/libs/testdiff/testdiff.go:23
                                                        /Users/denis.bilenko/work/cli/libs/testdiff/golden.go:43
                                                        /Users/denis.bilenko/work/cli/internal/testcli/golden.go:23
                                                        /Users/denis.bilenko/work/cli/integration/bundle/init_default_python_test.go:92
                                                        /Users/denis.bilenko/work/cli/integration/bundle/init_default_python_test.go:45
...
```

After:

```
+ go test --timeout 3h -v -run TestDefaultPython/3.9 ./integration/bundle/
=== RUN   TestDefaultPython
=== RUN   TestDefaultPython/3.9
    init_default_python_test.go:51: CLOUD_ENV=aws
    init_default_python_test.go:92:   args: bundle, init, default-python, --config-file, config.json
    init_default_python_test.go:92: stderr:
    init_default_python_test.go:92: stderr: Welcome to the default Python template for Databricks Asset Bundles!
...
    init_default_python_test.go:92:
                Error Trace:    /Users/denis.bilenko/work/cli/libs/testdiff/testdiff.go:24
                                                        /Users/denis.bilenko/work/cli/libs/testdiff/golden.go:46
                                                        /Users/denis.bilenko/work/cli/internal/testcli/golden.go:23
                                                        /Users/denis.bilenko/work/cli/integration/bundle/init_default_python_test.go:92
                                                        /Users/denis.bilenko/work/cli/integration/bundle/init_default_python_test.go:45
...
```
---
 integration/internal/acc/workspace.go | 14 +++++---
 internal/testcli/golden.go            |  5 +--
 internal/testcli/runner.go            | 47 ++++++++++++++++++++++++---
 internal/testutil/interface.go        |  2 ++
 libs/testdiff/golden.go               | 11 +++++--
 libs/testdiff/testdiff.go             |  4 ++-
 6 files changed, 69 insertions(+), 14 deletions(-)

diff --git a/integration/internal/acc/workspace.go b/integration/internal/acc/workspace.go
index 2f8a5b8e7..64deda7c1 100644
--- a/integration/internal/acc/workspace.go
+++ b/integration/internal/acc/workspace.go
@@ -21,9 +21,10 @@ type WorkspaceT struct {
 }
 
 func WorkspaceTest(t testutil.TestingT) (context.Context, *WorkspaceT) {
+	t.Helper()
 	loadDebugEnvIfRunFromIDE(t, "workspace")
 
-	t.Log(testutil.GetEnvOrSkipTest(t, "CLOUD_ENV"))
+	t.Logf("CLOUD_ENV=%s", testutil.GetEnvOrSkipTest(t, "CLOUD_ENV"))
 
 	w, err := databricks.NewWorkspaceClient()
 	require.NoError(t, err)
@@ -41,9 +42,10 @@ func WorkspaceTest(t testutil.TestingT) (context.Context, *WorkspaceT) {
 
 // Run the workspace test only on UC workspaces.
 func UcWorkspaceTest(t testutil.TestingT) (context.Context, *WorkspaceT) {
+	t.Helper()
 	loadDebugEnvIfRunFromIDE(t, "workspace")
 
-	t.Log(testutil.GetEnvOrSkipTest(t, "CLOUD_ENV"))
+	t.Logf("CLOUD_ENV=%s", testutil.GetEnvOrSkipTest(t, "CLOUD_ENV"))
 
 	if os.Getenv("TEST_METASTORE_ID") == "" {
 		t.Skipf("Skipping on non-UC workspaces")
@@ -67,19 +69,21 @@ func UcWorkspaceTest(t testutil.TestingT) (context.Context, *WorkspaceT) {
 }
 
 func (t *WorkspaceT) TestClusterID() string {
+	t.Helper()
 	clusterID := testutil.GetEnvOrSkipTest(t, "TEST_BRICKS_CLUSTER_ID")
 	err := t.W.Clusters.EnsureClusterIsRunning(t.ctx, clusterID)
-	require.NoError(t, err)
+	require.NoError(t, err, "Unexpected error from EnsureClusterIsRunning for clusterID=%s", clusterID)
 	return clusterID
 }
 
 func (t *WorkspaceT) RunPython(code string) (string, error) {
+	t.Helper()
 	var err error
 
 	// Create command executor only once per test.
 	if t.exec == nil {
 		t.exec, err = t.W.CommandExecution.Start(t.ctx, t.TestClusterID(), compute.LanguagePython)
-		require.NoError(t, err)
+		require.NoError(t, err, "Unexpected error from CommandExecution.Start(clusterID=%v)", t.TestClusterID())
 
 		t.Cleanup(func() {
 			err := t.exec.Destroy(t.ctx)
@@ -88,7 +92,7 @@ func (t *WorkspaceT) RunPython(code string) (string, error) {
 	}
 
 	results, err := t.exec.Execute(t.ctx, code)
-	require.NoError(t, err)
+	require.NoError(t, err, "Unexpected error from Execute(%v)", code)
 	require.NotEqual(t, compute.ResultTypeError, results.ResultType, results.Cause)
 	output, ok := results.Data.(string)
 	require.True(t, ok, "unexpected type %T", results.Data)
diff --git a/internal/testcli/golden.go b/internal/testcli/golden.go
index a16f42c72..669cc2f9b 100644
--- a/internal/testcli/golden.go
+++ b/internal/testcli/golden.go
@@ -3,7 +3,6 @@ package testcli
 import (
 	"context"
 	"fmt"
-	"strings"
 
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/testdiff"
@@ -11,7 +10,7 @@ import (
 )
 
 func captureOutput(t testutil.TestingT, ctx context.Context, args []string) string {
-	t.Logf("run args: [%s]", strings.Join(args, ", "))
+	t.Helper()
 	r := NewRunner(t, ctx, args...)
 	stdout, stderr, err := r.Run()
 	assert.NoError(t, err)
@@ -19,11 +18,13 @@ func captureOutput(t testutil.TestingT, ctx context.Context, args []string) stri
 }
 
 func AssertOutput(t testutil.TestingT, ctx context.Context, args []string, expectedPath string) {
+	t.Helper()
 	out := captureOutput(t, ctx, args)
 	testdiff.AssertOutput(t, ctx, out, fmt.Sprintf("Output from %v", args), expectedPath)
 }
 
 func AssertOutputJQ(t testutil.TestingT, ctx context.Context, args []string, expectedPath string, ignorePaths []string) {
+	t.Helper()
 	out := captureOutput(t, ctx, args)
 	testdiff.AssertOutputJQ(t, ctx, out, fmt.Sprintf("Output from %v", args), expectedPath, ignorePaths)
 }
diff --git a/internal/testcli/runner.go b/internal/testcli/runner.go
index 95073b57c..52decad2c 100644
--- a/internal/testcli/runner.go
+++ b/internal/testcli/runner.go
@@ -69,6 +69,7 @@ func consumeLines(ctx context.Context, wg *sync.WaitGroup, r io.Reader) <-chan s
 }
 
 func (r *Runner) registerFlagCleanup(c *cobra.Command) {
+	r.Helper()
 	// Find target command that will be run. Example: if the command run is `databricks fs cp`,
 	// target command corresponds to `cp`
 	targetCmd, _, err := c.Find(r.args)
@@ -230,13 +231,48 @@ func (r *Runner) RunBackground() {
 }
 
 func (r *Runner) Run() (bytes.Buffer, bytes.Buffer, error) {
-	r.RunBackground()
-	err := <-r.errch
-	return r.stdout, r.stderr, err
+	r.Helper()
+	var stdout, stderr bytes.Buffer
+	ctx := cmdio.NewContext(r.ctx, &cmdio.Logger{
+		Mode:   flags.ModeAppend,
+		Reader: bufio.Reader{},
+		Writer: &stderr,
+	})
+
+	cli := cmd.New(ctx)
+	cli.SetOut(&stdout)
+	cli.SetErr(&stderr)
+	cli.SetArgs(r.args)
+
+	r.Logf("  args: %s", strings.Join(r.args, ", "))
+
+	err := root.Execute(ctx, cli)
+	if err != nil {
+		r.Logf(" error: %s", err)
+	}
+
+	if stdout.Len() > 0 {
+		// Make a copy of the buffer such that it remains "unread".
+		scanner := bufio.NewScanner(bytes.NewBuffer(stdout.Bytes()))
+		for scanner.Scan() {
+			r.Logf("stdout: %s", scanner.Text())
+		}
+	}
+
+	if stderr.Len() > 0 {
+		// Make a copy of the buffer such that it remains "unread".
+		scanner := bufio.NewScanner(bytes.NewBuffer(stderr.Bytes()))
+		for scanner.Scan() {
+			r.Logf("stderr: %s", scanner.Text())
+		}
+	}
+
+	return stdout, stderr, err
 }
 
 // Like [require.Eventually] but errors if the underlying command has failed.
 func (r *Runner) Eventually(condition func() bool, waitFor, tick time.Duration, msgAndArgs ...any) {
+	r.Helper()
 	ch := make(chan bool, 1)
 
 	timer := time.NewTimer(waitFor)
@@ -269,12 +305,14 @@ func (r *Runner) Eventually(condition func() bool, waitFor, tick time.Duration,
 }
 
 func (r *Runner) RunAndExpectOutput(heredoc string) {
+	r.Helper()
 	stdout, _, err := r.Run()
 	require.NoError(r, err)
 	require.Equal(r, cmdio.Heredoc(heredoc), strings.TrimSpace(stdout.String()))
 }
 
 func (r *Runner) RunAndParseJSON(v any) {
+	r.Helper()
 	stdout, _, err := r.Run()
 	require.NoError(r, err)
 	err = json.Unmarshal(stdout.Bytes(), &v)
@@ -291,7 +329,7 @@ func NewRunner(t testutil.TestingT, ctx context.Context, args ...string) *Runner
 }
 
 func RequireSuccessfulRun(t testutil.TestingT, ctx context.Context, args ...string) (bytes.Buffer, bytes.Buffer) {
-	t.Logf("run args: [%s]", strings.Join(args, ", "))
+	t.Helper()
 	r := NewRunner(t, ctx, args...)
 	stdout, stderr, err := r.Run()
 	require.NoError(t, err)
@@ -299,6 +337,7 @@ func RequireSuccessfulRun(t testutil.TestingT, ctx context.Context, args ...stri
 }
 
 func RequireErrorRun(t testutil.TestingT, ctx context.Context, args ...string) (bytes.Buffer, bytes.Buffer, error) {
+	t.Helper()
 	r := NewRunner(t, ctx, args...)
 	stdout, stderr, err := r.Run()
 	require.Error(t, err)
diff --git a/internal/testutil/interface.go b/internal/testutil/interface.go
index 2c3004800..97441212d 100644
--- a/internal/testutil/interface.go
+++ b/internal/testutil/interface.go
@@ -24,4 +24,6 @@ type TestingT interface {
 	Setenv(key, value string)
 
 	TempDir() string
+
+	Helper()
 }
diff --git a/libs/testdiff/golden.go b/libs/testdiff/golden.go
index c338ac9e5..b67eb50a9 100644
--- a/libs/testdiff/golden.go
+++ b/libs/testdiff/golden.go
@@ -19,22 +19,25 @@ import (
 var OverwriteMode = os.Getenv("TESTS_OUTPUT") == "OVERWRITE"
 
 func ReadFile(t testutil.TestingT, ctx context.Context, filename string) string {
+	t.Helper()
 	data, err := os.ReadFile(filename)
 	if os.IsNotExist(err) {
 		return ""
 	}
-	assert.NoError(t, err)
+	assert.NoError(t, err, "Failed to read %s", filename)
 	// On CI, on Windows \n in the file somehow end up as \r\n
 	return NormalizeNewlines(string(data))
 }
 
 func WriteFile(t testutil.TestingT, filename, data string) {
+	t.Helper()
 	t.Logf("Overwriting %s", filename)
 	err := os.WriteFile(filename, []byte(data), 0o644)
-	assert.NoError(t, err)
+	assert.NoError(t, err, "Failed to write %s", filename)
 }
 
 func AssertOutput(t testutil.TestingT, ctx context.Context, out, outTitle, expectedPath string) {
+	t.Helper()
 	expected := ReadFile(t, ctx, expectedPath)
 
 	out = ReplaceOutput(t, ctx, out)
@@ -49,6 +52,7 @@ func AssertOutput(t testutil.TestingT, ctx context.Context, out, outTitle, expec
 }
 
 func AssertOutputJQ(t testutil.TestingT, ctx context.Context, out, outTitle, expectedPath string, ignorePaths []string) {
+	t.Helper()
 	expected := ReadFile(t, ctx, expectedPath)
 
 	out = ReplaceOutput(t, ctx, out)
@@ -69,6 +73,7 @@ var (
 )
 
 func ReplaceOutput(t testutil.TestingT, ctx context.Context, out string) string {
+	t.Helper()
 	out = NormalizeNewlines(out)
 	replacements := GetReplacementsMap(ctx)
 	if replacements == nil {
@@ -136,6 +141,7 @@ func GetReplacementsMap(ctx context.Context) *ReplacementsContext {
 }
 
 func PrepareReplacements(t testutil.TestingT, r *ReplacementsContext, w *databricks.WorkspaceClient) {
+	t.Helper()
 	// in some clouds (gcp) w.Config.Host includes "https://" prefix in others it's really just a host (azure)
 	host := strings.TrimPrefix(strings.TrimPrefix(w.Config.Host, "http://"), "https://")
 	r.Set(host, "$DATABRICKS_HOST")
@@ -167,6 +173,7 @@ func PrepareReplacements(t testutil.TestingT, r *ReplacementsContext, w *databri
 }
 
 func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam.User) {
+	t.Helper()
 	// There could be exact matches or overlap between different name fields, so sort them by length
 	// to ensure we match the largest one first and map them all to the same token
 	names := []string{
diff --git a/libs/testdiff/testdiff.go b/libs/testdiff/testdiff.go
index 1e1df727a..71b781362 100644
--- a/libs/testdiff/testdiff.go
+++ b/libs/testdiff/testdiff.go
@@ -18,9 +18,10 @@ func UnifiedDiff(filename1, filename2, s1, s2 string) string {
 }
 
 func AssertEqualTexts(t testutil.TestingT, filename1, filename2, expected, out string) {
+	t.Helper()
 	if len(out) < 1000 && len(expected) < 1000 {
 		// This shows full strings + diff which could be useful when debugging newlines
-		assert.Equal(t, expected, out)
+		assert.Equal(t, expected, out, "%s vs %s", filename1, filename2)
 	} else {
 		// only show diff for large texts
 		diff := UnifiedDiff(filename1, filename2, expected, out)
@@ -29,6 +30,7 @@ func AssertEqualTexts(t testutil.TestingT, filename1, filename2, expected, out s
 }
 
 func AssertEqualJQ(t testutil.TestingT, expectedName, outName, expected, out string, ignorePaths []string) {
+	t.Helper()
 	patch, err := jsondiff.CompareJSON([]byte(expected), []byte(out))
 	if err != nil {
 		t.Logf("CompareJSON error for %s vs %s: %s (fallback to textual comparison)", outName, expectedName, err)

From d7f69f6a5dc80450498119a7e2a60eecfc3f0cab Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 2 Jan 2025 11:31:35 +0100
Subject: [PATCH 009/247] Upgrade golangci-lint to v1.63.1 (#2064)

Upgrade your laptops with: brew install golangci-lint

This has a lot more autofixes, which makes it easier to adopt those
linters.

https://golangci-lint.run/product/changelog/#v1630
---
 .github/workflows/push.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
index a51927594..c737993aa 100644
--- a/.github/workflows/push.yml
+++ b/.github/workflows/push.yml
@@ -75,7 +75,7 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v6
         with:
-          version: v1.62.2
+          version: v1.63.1
           args: --timeout=15m
 
   validate-bundle-schema:

From c262b75c3d0e48839fe327697e2758d3120d780e Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 2 Jan 2025 11:33:06 +0100
Subject: [PATCH 010/247] Make lint.sh to run golangci-lint only once in the
 best case (#2062)

Follow up to #2051 and #2056.

Running golangci-lint twice always is measurably slower (1.5s vs 2.5s),
so only run it twice in case it is necessary.
---
 lint.sh | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/lint.sh b/lint.sh
index 13efa9855..1f881eaf7 100755
--- a/lint.sh
+++ b/lint.sh
@@ -1,9 +1,14 @@
 #!/bin/bash
-set -euo pipefail
+set -uo pipefail
 # With golangci-lint, if there are any compliation issues, then formatters' autofix won't be applied.
 # https://github.com/golangci/golangci-lint/issues/5257
-# However, running goimports first alone will actually fix some of the compilation issues.
-# Fixing formatting is also reasonable thing to do.
-# For this reason, this script runs golangci-lint in two stages:
-golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix $@
-exec golangci-lint run --fix $@
+
+golangci-lint run --fix "$@"
+lint_exit_code=$?
+
+if [ $lint_exit_code -ne 0 ]; then
+    # These linters work in presence of compilation issues when run alone, so let's get these fixes at least.
+    golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix "$@"
+fi
+
+exit $lint_exit_code

From b053bfb4de577d8ee2d4058672a694af69d80891 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Thu, 2 Jan 2025 11:41:55 +0100
Subject: [PATCH 011/247] Verify that the bundle schema is up to date in CI
 (#2061)

## Changes

I noticed a diff in the schema in #2052.

This check should be performed automatically.

## Tests

This PR includes a commit that changes the schema to check that the
workflow actually fails.
---
 .github/workflows/push.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
index c737993aa..25bd5f4d6 100644
--- a/.github/workflows/push.yml
+++ b/.github/workflows/push.yml
@@ -90,6 +90,13 @@ jobs:
         with:
           go-version: 1.23.4
 
+      - name: Verify that the schema is up to date
+        run: |
+          if ! ( make schema && git diff --exit-code ); then
+            echo "The schema is not up to date. Please run 'make schema' and commit the changes."
+            exit 1
+          fi
+
       # Github repo: https://github.com/ajv-validator/ajv-cli
       - name: Install ajv-cli
         run: npm install -g ajv-cli@5.0.0

From 0b80784df759673d0ccc3874d19d6662ff11ac54 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 2 Jan 2025 12:03:41 +0100
Subject: [PATCH 012/247] Enable testifylint and fix the issues (#2065)

## Changes
- Enable new linter: testifylint.
- Apply fixes with --fix.
- Fix remaining issues (mostly with aider).

There were 2 cases we --fix did the wrong thing - this seems to a be a
bug in linter: https://github.com/Antonboom/testifylint/issues/210

Nonetheless, I kept that check enabled, it seems useful, just need to be
fixed manually after autofix.

## Tests
Existing tests
---
 .golangci.yaml                                |  8 +++-
 bundle/bundle_test.go                         |  3 +-
 .../configure_dashboard_defaults_test.go      |  6 +--
 .../config/mutator/default_queueing_test.go   | 10 ++--
 .../mutator/environments_compat_test.go       |  4 +-
 bundle/config/mutator/merge_job_tasks_test.go |  4 +-
 .../mutator/process_target_mode_test.go       | 12 ++---
 .../resolve_variable_references_test.go       |  6 +--
 .../mutator/rewrite_workspace_prefix_test.go  |  2 +-
 bundle/config/mutator/set_variables_test.go   |  8 ++--
 bundle/config/resources_test.go               |  6 +--
 bundle/config/validate/files_to_sync_test.go  |  4 +-
 .../validate/job_cluster_key_defined_test.go  | 10 ++--
 bundle/context_test.go                        |  2 +-
 bundle/deploy/state_pull_test.go              |  3 +-
 bundle/deploy/state_update_test.go            | 12 ++---
 bundle/deploy/terraform/convert_test.go       | 10 ++--
 bundle/deploy/terraform/init_test.go          |  4 +-
 bundle/internal/schema/main_test.go           |  3 +-
 bundle/libraries/filer_volume_test.go         |  4 +-
 bundle/libraries/helpers_test.go              | 12 ++---
 bundle/permissions/filter_test.go             | 28 +++++------
 bundle/permissions/mutator_test.go            |  2 +-
 .../permission_diagnostics_test.go            |  2 +-
 bundle/run/job_test.go                        |  2 +-
 bundle/tests/complex_variables_test.go        |  2 +-
 bundle/tests/environment_git_test.go          |  5 +-
 bundle/tests/environment_overrides_test.go    |  8 ++--
 .../environments_job_and_pipeline_test.go     |  6 +--
 bundle/tests/git_test.go                      |  5 +-
 bundle/tests/issue_1828_test.go               |  4 +-
 bundle/tests/job_and_pipeline_test.go         |  6 +--
 bundle/tests/job_cluster_key_test.go          |  6 +--
 bundle/tests/model_serving_endpoint_test.go   |  2 +-
 bundle/tests/override_job_tasks_test.go       | 28 +++++------
 bundle/tests/presets_test.go                  |  4 +-
 bundle/tests/python_wheel_test.go             | 10 ++--
 bundle/tests/quality_monitor_test.go          |  2 +-
 bundle/tests/registered_model_test.go         |  2 +-
 .../sync_include_exclude_no_matches_test.go   | 20 ++++----
 bundle/tests/sync_test.go                     |  4 +-
 cmd/bundle/generate/dashboard_test.go         |  2 +-
 cmd/bundle/generate/generate_test.go          |  3 +-
 cmd/configure/configure_test.go               |  4 +-
 cmd/labs/github/ref_test.go                   |  9 ++--
 cmd/labs/github/releases_test.go              |  5 +-
 cmd/labs/github/repositories_test.go          |  5 +-
 cmd/labs/localcache/jsonfile_test.go          |  2 +-
 cmd/labs/project/installer_test.go            | 17 +++----
 integration/bundle/artifacts_test.go          | 12 ++---
 integration/bundle/deploy_test.go             |  3 +-
 integration/bundle/destroy_test.go            |  5 +-
 integration/cmd/fs/ls_test.go                 | 15 +++---
 integration/cmd/fs/mkdir_test.go              |  8 ++--
 integration/cmd/sync/sync_test.go             |  2 +-
 integration/libs/filer/filer_test.go          | 46 +++++++++----------
 libs/auth/cache/file_test.go                  |  2 +-
 libs/databrickscfg/ops_test.go                |  2 +-
 libs/databrickscfg/profile/file_test.go       |  8 ++--
 libs/errs/aggregate_test.go                   | 19 ++++----
 libs/filer/completer/completer_test.go        | 12 ++---
 libs/filer/fs_test.go                         |  4 +-
 libs/fileset/glob_test.go                     |  2 +-
 libs/flags/json_flag_test.go                  |  8 ++--
 libs/folders/folders_test.go                  |  2 +-
 libs/jsonschema/utils_test.go                 |  4 +-
 libs/notebook/detect_test.go                  |  5 +-
 libs/process/background_test.go               |  2 +-
 libs/process/forwarded_test.go                |  4 +-
 libs/process/opts_test.go                     |  2 +-
 libs/python/interpreters_unix_test.go         |  2 +-
 libs/sync/snapshot_test.go                    | 24 +++++-----
 libs/sync/sync_test.go                        | 12 ++---
 libs/template/config_test.go                  |  8 ++--
 libs/template/helpers_test.go                 |  2 +-
 libs/template/renderer_test.go                |  8 ++--
 libs/vfs/filer_test.go                        |  9 ++--
 77 files changed, 277 insertions(+), 283 deletions(-)

diff --git a/.golangci.yaml b/.golangci.yaml
index 9e69e5146..fd83e0882 100644
--- a/.golangci.yaml
+++ b/.golangci.yaml
@@ -11,6 +11,7 @@ linters:
     - gofmt
     - gofumpt
     - goimports
+    - testifylint
 linters-settings:
   govet:
     enable-all: true
@@ -32,7 +33,10 @@ linters-settings:
   gofumpt:
     module-path: github.com/databricks/cli
     extra-rules: true
-  #goimports:
-  #  local-prefixes: github.com/databricks/cli
+  testifylint:
+    enable-all: true
+    disable:
+      # good check, but we have too many assert.(No)?Errorf? so excluding for now
+      - require-error
 issues:
   exclude-dirs-use-default: false  # recommended by docs https://golangci-lint.run/usage/false-positives/
diff --git a/bundle/bundle_test.go b/bundle/bundle_test.go
index 1c3102357..d52088e34 100644
--- a/bundle/bundle_test.go
+++ b/bundle/bundle_test.go
@@ -2,7 +2,6 @@ package bundle
 
 import (
 	"context"
-	"errors"
 	"io/fs"
 	"os"
 	"path/filepath"
@@ -16,7 +15,7 @@ import (
 
 func TestLoadNotExists(t *testing.T) {
 	b, err := Load(context.Background(), "/doesntexist")
-	assert.True(t, errors.Is(err, fs.ErrNotExist))
+	assert.ErrorIs(t, err, fs.ErrNotExist)
 	assert.Nil(t, b)
 }
 
diff --git a/bundle/config/mutator/configure_dashboard_defaults_test.go b/bundle/config/mutator/configure_dashboard_defaults_test.go
index 2234f9a73..9794d355c 100644
--- a/bundle/config/mutator/configure_dashboard_defaults_test.go
+++ b/bundle/config/mutator/configure_dashboard_defaults_test.go
@@ -109,19 +109,19 @@ func TestConfigureDashboardDefaultsEmbedCredentials(t *testing.T) {
 	// Set to true; still true.
 	v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d1.embed_credentials")
 	if assert.NoError(t, err) {
-		assert.Equal(t, true, v.MustBool())
+		assert.True(t, v.MustBool())
 	}
 
 	// Set to false; still false.
 	v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d2.embed_credentials")
 	if assert.NoError(t, err) {
-		assert.Equal(t, false, v.MustBool())
+		assert.False(t, v.MustBool())
 	}
 
 	// Not set; now false.
 	v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d3.embed_credentials")
 	if assert.NoError(t, err) {
-		assert.Equal(t, false, v.MustBool())
+		assert.False(t, v.MustBool())
 	}
 
 	// No valid dashboard; no change.
diff --git a/bundle/config/mutator/default_queueing_test.go b/bundle/config/mutator/default_queueing_test.go
index d3621663b..4c521812e 100644
--- a/bundle/config/mutator/default_queueing_test.go
+++ b/bundle/config/mutator/default_queueing_test.go
@@ -28,8 +28,8 @@ func TestDefaultQueueingApplyNoJobs(t *testing.T) {
 		},
 	}
 	d := bundle.Apply(context.Background(), b, DefaultQueueing())
-	assert.Len(t, d, 0)
-	assert.Len(t, b.Config.Resources.Jobs, 0)
+	assert.Empty(t, d)
+	assert.Empty(t, b.Config.Resources.Jobs)
 }
 
 func TestDefaultQueueingApplyJobsAlreadyEnabled(t *testing.T) {
@@ -47,7 +47,7 @@ func TestDefaultQueueingApplyJobsAlreadyEnabled(t *testing.T) {
 		},
 	}
 	d := bundle.Apply(context.Background(), b, DefaultQueueing())
-	assert.Len(t, d, 0)
+	assert.Empty(t, d)
 	assert.True(t, b.Config.Resources.Jobs["job"].Queue.Enabled)
 }
 
@@ -66,7 +66,7 @@ func TestDefaultQueueingApplyEnableQueueing(t *testing.T) {
 		},
 	}
 	d := bundle.Apply(context.Background(), b, DefaultQueueing())
-	assert.Len(t, d, 0)
+	assert.Empty(t, d)
 	assert.NotNil(t, b.Config.Resources.Jobs["job"].Queue)
 	assert.True(t, b.Config.Resources.Jobs["job"].Queue.Enabled)
 }
@@ -96,7 +96,7 @@ func TestDefaultQueueingApplyWithMultipleJobs(t *testing.T) {
 		},
 	}
 	d := bundle.Apply(context.Background(), b, DefaultQueueing())
-	assert.Len(t, d, 0)
+	assert.Empty(t, d)
 	assert.False(t, b.Config.Resources.Jobs["job1"].Queue.Enabled)
 	assert.True(t, b.Config.Resources.Jobs["job2"].Queue.Enabled)
 	assert.True(t, b.Config.Resources.Jobs["job3"].Queue.Enabled)
diff --git a/bundle/config/mutator/environments_compat_test.go b/bundle/config/mutator/environments_compat_test.go
index 8a2129847..11facf9fb 100644
--- a/bundle/config/mutator/environments_compat_test.go
+++ b/bundle/config/mutator/environments_compat_test.go
@@ -44,7 +44,7 @@ func TestEnvironmentsToTargetsWithEnvironmentsDefined(t *testing.T) {
 
 	diags := bundle.Apply(context.Background(), b, mutator.EnvironmentsToTargets())
 	require.NoError(t, diags.Error())
-	assert.Len(t, b.Config.Environments, 0)
+	assert.Empty(t, b.Config.Environments)
 	assert.Len(t, b.Config.Targets, 1)
 }
 
@@ -61,6 +61,6 @@ func TestEnvironmentsToTargetsWithTargetsDefined(t *testing.T) {
 
 	diags := bundle.Apply(context.Background(), b, mutator.EnvironmentsToTargets())
 	require.NoError(t, diags.Error())
-	assert.Len(t, b.Config.Environments, 0)
+	assert.Empty(t, b.Config.Environments)
 	assert.Len(t, b.Config.Targets, 1)
 }
diff --git a/bundle/config/mutator/merge_job_tasks_test.go b/bundle/config/mutator/merge_job_tasks_test.go
index a9dae1e10..e6675eefb 100644
--- a/bundle/config/mutator/merge_job_tasks_test.go
+++ b/bundle/config/mutator/merge_job_tasks_test.go
@@ -74,8 +74,8 @@ func TestMergeJobTasks(t *testing.T) {
 	assert.Equal(t, "i3.2xlarge", cluster.NodeTypeId)
 	assert.Equal(t, 4, cluster.NumWorkers)
 	assert.Len(t, task0.Libraries, 2)
-	assert.Equal(t, task0.Libraries[0].Whl, "package1")
-	assert.Equal(t, task0.Libraries[1].Pypi.Package, "package2")
+	assert.Equal(t, "package1", task0.Libraries[0].Whl)
+	assert.Equal(t, "package2", task0.Libraries[1].Pypi.Package)
 
 	// This task was left untouched.
 	task1 := j.Tasks[1].NewCluster
diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go
index 14d524416..e1aa9e59b 100644
--- a/bundle/config/mutator/process_target_mode_test.go
+++ b/bundle/config/mutator/process_target_mode_test.go
@@ -163,18 +163,18 @@ func TestProcessTargetModeDevelopment(t *testing.T) {
 
 	// Job 1
 	assert.Equal(t, "[dev lennart] job1", b.Config.Resources.Jobs["job1"].Name)
-	assert.Equal(t, b.Config.Resources.Jobs["job1"].Tags["existing"], "tag")
-	assert.Equal(t, b.Config.Resources.Jobs["job1"].Tags["dev"], "lennart")
-	assert.Equal(t, b.Config.Resources.Jobs["job1"].Schedule.PauseStatus, jobs.PauseStatusPaused)
+	assert.Equal(t, "tag", b.Config.Resources.Jobs["job1"].Tags["existing"])
+	assert.Equal(t, "lennart", b.Config.Resources.Jobs["job1"].Tags["dev"])
+	assert.Equal(t, jobs.PauseStatusPaused, b.Config.Resources.Jobs["job1"].Schedule.PauseStatus)
 
 	// Job 2
 	assert.Equal(t, "[dev lennart] job2", b.Config.Resources.Jobs["job2"].Name)
-	assert.Equal(t, b.Config.Resources.Jobs["job2"].Tags["dev"], "lennart")
-	assert.Equal(t, b.Config.Resources.Jobs["job2"].Schedule.PauseStatus, jobs.PauseStatusUnpaused)
+	assert.Equal(t, "lennart", b.Config.Resources.Jobs["job2"].Tags["dev"])
+	assert.Equal(t, jobs.PauseStatusUnpaused, b.Config.Resources.Jobs["job2"].Schedule.PauseStatus)
 
 	// Pipeline 1
 	assert.Equal(t, "[dev lennart] pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name)
-	assert.Equal(t, false, b.Config.Resources.Pipelines["pipeline1"].Continuous)
+	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].Continuous)
 	assert.True(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
 
 	// Experiment 1
diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go
index 7bb6f11a0..07972ecf4 100644
--- a/bundle/config/mutator/resolve_variable_references_test.go
+++ b/bundle/config/mutator/resolve_variable_references_test.go
@@ -185,11 +185,11 @@ func TestResolveVariableReferencesForPrimitiveNonStringFields(t *testing.T) {
 	// Apply for the variable prefix. This should resolve the variables to their values.
 	diags = bundle.Apply(context.Background(), b, ResolveVariableReferences("variables"))
 	require.NoError(t, diags.Error())
-	assert.Equal(t, true, b.Config.Resources.Jobs["job1"].JobSettings.NotificationSettings.NoAlertForCanceledRuns)
-	assert.Equal(t, true, b.Config.Resources.Jobs["job1"].JobSettings.NotificationSettings.NoAlertForSkippedRuns)
+	assert.True(t, b.Config.Resources.Jobs["job1"].JobSettings.NotificationSettings.NoAlertForCanceledRuns)
+	assert.True(t, b.Config.Resources.Jobs["job1"].JobSettings.NotificationSettings.NoAlertForSkippedRuns)
 	assert.Equal(t, 1, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.Autoscale.MinWorkers)
 	assert.Equal(t, 2, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.Autoscale.MaxWorkers)
-	assert.Equal(t, 0.5, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.AzureAttributes.SpotBidMaxPrice)
+	assert.InDelta(t, 0.5, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.AzureAttributes.SpotBidMaxPrice, 0.0001)
 }
 
 func TestResolveComplexVariable(t *testing.T) {
diff --git a/bundle/config/mutator/rewrite_workspace_prefix_test.go b/bundle/config/mutator/rewrite_workspace_prefix_test.go
index 48973a4cf..099738c02 100644
--- a/bundle/config/mutator/rewrite_workspace_prefix_test.go
+++ b/bundle/config/mutator/rewrite_workspace_prefix_test.go
@@ -71,7 +71,7 @@ func TestNoWorkspacePrefixUsed(t *testing.T) {
 	}
 
 	for _, d := range diags {
-		require.Equal(t, d.Severity, diag.Warning)
+		require.Equal(t, diag.Warning, d.Severity)
 		require.Contains(t, expectedErrors, d.Summary)
 		delete(expectedErrors, d.Summary)
 	}
diff --git a/bundle/config/mutator/set_variables_test.go b/bundle/config/mutator/set_variables_test.go
index d9719793f..07a5c8214 100644
--- a/bundle/config/mutator/set_variables_test.go
+++ b/bundle/config/mutator/set_variables_test.go
@@ -30,7 +30,7 @@ func TestSetVariableFromProcessEnvVar(t *testing.T) {
 
 	err = convert.ToTyped(&variable, v)
 	require.NoError(t, err)
-	assert.Equal(t, variable.Value, "process-env")
+	assert.Equal(t, "process-env", variable.Value)
 }
 
 func TestSetVariableUsingDefaultValue(t *testing.T) {
@@ -48,7 +48,7 @@ func TestSetVariableUsingDefaultValue(t *testing.T) {
 
 	err = convert.ToTyped(&variable, v)
 	require.NoError(t, err)
-	assert.Equal(t, variable.Value, "default")
+	assert.Equal(t, "default", variable.Value)
 }
 
 func TestSetVariableWhenAlreadyAValueIsAssigned(t *testing.T) {
@@ -70,7 +70,7 @@ func TestSetVariableWhenAlreadyAValueIsAssigned(t *testing.T) {
 
 	err = convert.ToTyped(&variable, v)
 	require.NoError(t, err)
-	assert.Equal(t, variable.Value, "assigned-value")
+	assert.Equal(t, "assigned-value", variable.Value)
 }
 
 func TestSetVariableEnvVarValueDoesNotOverridePresetValue(t *testing.T) {
@@ -95,7 +95,7 @@ func TestSetVariableEnvVarValueDoesNotOverridePresetValue(t *testing.T) {
 
 	err = convert.ToTyped(&variable, v)
 	require.NoError(t, err)
-	assert.Equal(t, variable.Value, "assigned-value")
+	assert.Equal(t, "assigned-value", variable.Value)
 }
 
 func TestSetVariablesErrorsIfAValueCouldNotBeResolved(t *testing.T) {
diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go
index 2d05acf3e..3da645585 100644
--- a/bundle/config/resources_test.go
+++ b/bundle/config/resources_test.go
@@ -37,11 +37,11 @@ func TestCustomMarshallerIsImplemented(t *testing.T) {
 		field := rt.Field(i)
 
 		// Fields in Resources are expected be of the form map[string]*resourceStruct
-		assert.Equal(t, field.Type.Kind(), reflect.Map, "Resource %s is not a map", field.Name)
+		assert.Equal(t, reflect.Map, field.Type.Kind(), "Resource %s is not a map", field.Name)
 		kt := field.Type.Key()
-		assert.Equal(t, kt.Kind(), reflect.String, "Resource %s is not a map with string keys", field.Name)
+		assert.Equal(t, reflect.String, kt.Kind(), "Resource %s is not a map with string keys", field.Name)
 		vt := field.Type.Elem()
-		assert.Equal(t, vt.Kind(), reflect.Ptr, "Resource %s is not a map with pointer values", field.Name)
+		assert.Equal(t, reflect.Ptr, vt.Kind(), "Resource %s is not a map with pointer values", field.Name)
 
 		// Marshalling a resourceStruct will panic if resourceStruct does not have a custom marshaller
 		// This is because resourceStruct embeds a Go SDK struct that implements
diff --git a/bundle/config/validate/files_to_sync_test.go b/bundle/config/validate/files_to_sync_test.go
index d6a1ed59a..dd40295c3 100644
--- a/bundle/config/validate/files_to_sync_test.go
+++ b/bundle/config/validate/files_to_sync_test.go
@@ -87,7 +87,7 @@ func TestFilesToSync_EverythingIgnored(t *testing.T) {
 	ctx := context.Background()
 	rb := bundle.ReadOnly(b)
 	diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync())
-	require.Equal(t, 1, len(diags))
+	require.Len(t, diags, 1)
 	assert.Equal(t, diag.Warning, diags[0].Severity)
 	assert.Equal(t, "There are no files to sync, please check your .gitignore", diags[0].Summary)
 }
@@ -101,7 +101,7 @@ func TestFilesToSync_EverythingExcluded(t *testing.T) {
 	ctx := context.Background()
 	rb := bundle.ReadOnly(b)
 	diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync())
-	require.Equal(t, 1, len(diags))
+	require.Len(t, diags, 1)
 	assert.Equal(t, diag.Warning, diags[0].Severity)
 	assert.Equal(t, "There are no files to sync, please check your .gitignore and sync.exclude configuration", diags[0].Summary)
 }
diff --git a/bundle/config/validate/job_cluster_key_defined_test.go b/bundle/config/validate/job_cluster_key_defined_test.go
index 176b0fedc..2cbdb7c6a 100644
--- a/bundle/config/validate/job_cluster_key_defined_test.go
+++ b/bundle/config/validate/job_cluster_key_defined_test.go
@@ -34,7 +34,7 @@ func TestJobClusterKeyDefined(t *testing.T) {
 	}
 
 	diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobClusterKeyDefined())
-	require.Len(t, diags, 0)
+	require.Empty(t, diags)
 	require.NoError(t, diags.Error())
 }
 
@@ -59,8 +59,8 @@ func TestJobClusterKeyNotDefined(t *testing.T) {
 	diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobClusterKeyDefined())
 	require.Len(t, diags, 1)
 	require.NoError(t, diags.Error())
-	require.Equal(t, diags[0].Severity, diag.Warning)
-	require.Equal(t, diags[0].Summary, "job_cluster_key do-not-exist is not defined")
+	require.Equal(t, diag.Warning, diags[0].Severity)
+	require.Equal(t, "job_cluster_key do-not-exist is not defined", diags[0].Summary)
 }
 
 func TestJobClusterKeyDefinedInDifferentJob(t *testing.T) {
@@ -92,6 +92,6 @@ func TestJobClusterKeyDefinedInDifferentJob(t *testing.T) {
 	diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobClusterKeyDefined())
 	require.Len(t, diags, 1)
 	require.NoError(t, diags.Error())
-	require.Equal(t, diags[0].Severity, diag.Warning)
-	require.Equal(t, diags[0].Summary, "job_cluster_key do-not-exist is not defined")
+	require.Equal(t, diag.Warning, diags[0].Severity)
+	require.Equal(t, "job_cluster_key do-not-exist is not defined", diags[0].Summary)
 }
diff --git a/bundle/context_test.go b/bundle/context_test.go
index 3a0f159d9..89a6df052 100644
--- a/bundle/context_test.go
+++ b/bundle/context_test.go
@@ -12,7 +12,7 @@ func TestGetPanics(t *testing.T) {
 	defer func() {
 		r := recover()
 		require.NotNil(t, r, "The function did not panic")
-		assert.Equal(t, r, "context not configured with bundle")
+		assert.Equal(t, "context not configured with bundle", r)
 	}()
 
 	Get(context.Background())
diff --git a/bundle/deploy/state_pull_test.go b/bundle/deploy/state_pull_test.go
index 36c49fb01..f38b71f6b 100644
--- a/bundle/deploy/state_pull_test.go
+++ b/bundle/deploy/state_pull_test.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"errors"
 	"io"
 	"io/fs"
 	"os"
@@ -279,7 +278,7 @@ func TestStatePullNoState(t *testing.T) {
 	require.NoError(t, err)
 
 	_, err = os.Stat(statePath)
-	require.True(t, errors.Is(err, fs.ErrNotExist))
+	require.ErrorIs(t, err, fs.ErrNotExist)
 }
 
 func TestStatePullOlderState(t *testing.T) {
diff --git a/bundle/deploy/state_update_test.go b/bundle/deploy/state_update_test.go
index e561f534e..04c5579a8 100644
--- a/bundle/deploy/state_update_test.go
+++ b/bundle/deploy/state_update_test.go
@@ -60,7 +60,7 @@ func TestStateUpdate(t *testing.T) {
 	require.NoError(t, err)
 
 	require.Equal(t, int64(1), state.Seq)
-	require.Equal(t, state.Files, Filelist{
+	require.Equal(t, Filelist{
 		{
 			LocalPath: "test1.py",
 		},
@@ -68,7 +68,7 @@ func TestStateUpdate(t *testing.T) {
 			LocalPath:  "test2.py",
 			IsNotebook: true,
 		},
-	})
+	}, state.Files)
 	require.Equal(t, build.GetInfo().Version, state.CliVersion)
 
 	diags = bundle.Apply(ctx, b, s)
@@ -79,7 +79,7 @@ func TestStateUpdate(t *testing.T) {
 	require.NoError(t, err)
 
 	require.Equal(t, int64(2), state.Seq)
-	require.Equal(t, state.Files, Filelist{
+	require.Equal(t, Filelist{
 		{
 			LocalPath: "test1.py",
 		},
@@ -87,7 +87,7 @@ func TestStateUpdate(t *testing.T) {
 			LocalPath:  "test2.py",
 			IsNotebook: true,
 		},
-	})
+	}, state.Files)
 	require.Equal(t, build.GetInfo().Version, state.CliVersion)
 
 	// Valid non-empty UUID is generated.
@@ -130,7 +130,7 @@ func TestStateUpdateWithExistingState(t *testing.T) {
 	require.NoError(t, err)
 
 	require.Equal(t, int64(11), state.Seq)
-	require.Equal(t, state.Files, Filelist{
+	require.Equal(t, Filelist{
 		{
 			LocalPath: "test1.py",
 		},
@@ -138,7 +138,7 @@ func TestStateUpdateWithExistingState(t *testing.T) {
 			LocalPath:  "test2.py",
 			IsNotebook: true,
 		},
-	})
+	}, state.Files)
 	require.Equal(t, build.GetInfo().Version, state.CliVersion)
 
 	// Existing UUID is not overwritten.
diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go
index 61f26c088..84b3c5788 100644
--- a/bundle/deploy/terraform/convert_test.go
+++ b/bundle/deploy/terraform/convert_test.go
@@ -254,10 +254,10 @@ func TestBundleToTerraformPipeline(t *testing.T) {
 	assert.Equal(t, "my pipeline", resource.Name)
 	assert.Len(t, resource.Library, 2)
 	assert.Len(t, resource.Notification, 2)
-	assert.Equal(t, resource.Notification[0].Alerts, []string{"on-update-fatal-failure"})
-	assert.Equal(t, resource.Notification[0].EmailRecipients, []string{"jane@doe.com"})
-	assert.Equal(t, resource.Notification[1].Alerts, []string{"on-update-failure", "on-flow-failure"})
-	assert.Equal(t, resource.Notification[1].EmailRecipients, []string{"jane@doe.com", "john@doe.com"})
+	assert.Equal(t, []string{"on-update-fatal-failure"}, resource.Notification[0].Alerts)
+	assert.Equal(t, []string{"jane@doe.com"}, resource.Notification[0].EmailRecipients)
+	assert.Equal(t, []string{"on-update-failure", "on-flow-failure"}, resource.Notification[1].Alerts)
+	assert.Equal(t, []string{"jane@doe.com", "john@doe.com"}, resource.Notification[1].EmailRecipients)
 	assert.Nil(t, out.Data)
 }
 
@@ -454,7 +454,7 @@ func TestBundleToTerraformModelServing(t *testing.T) {
 	assert.Equal(t, "name", resource.Name)
 	assert.Equal(t, "model_name", resource.Config.ServedModels[0].ModelName)
 	assert.Equal(t, "1", resource.Config.ServedModels[0].ModelVersion)
-	assert.Equal(t, true, resource.Config.ServedModels[0].ScaleToZeroEnabled)
+	assert.True(t, resource.Config.ServedModels[0].ScaleToZeroEnabled)
 	assert.Equal(t, "Small", resource.Config.ServedModels[0].WorkloadSize)
 	assert.Equal(t, "model_name-1", resource.Config.TrafficConfig.Routes[0].ServedModelName)
 	assert.Equal(t, 100, resource.Config.TrafficConfig.Routes[0].TrafficPercentage)
diff --git a/bundle/deploy/terraform/init_test.go b/bundle/deploy/terraform/init_test.go
index a1ffc5a1a..30ac9e301 100644
--- a/bundle/deploy/terraform/init_test.go
+++ b/bundle/deploy/terraform/init_test.go
@@ -225,7 +225,7 @@ func TestSetProxyEnvVars(t *testing.T) {
 	env := make(map[string]string, 0)
 	err := setProxyEnvVars(context.Background(), env, b)
 	require.NoError(t, err)
-	assert.Len(t, env, 0)
+	assert.Empty(t, env)
 
 	// Lower case set.
 	clearEnv()
@@ -293,7 +293,7 @@ func TestSetUserProfileFromInheritEnvVars(t *testing.T) {
 	require.NoError(t, err)
 
 	assert.Contains(t, env, "USERPROFILE")
-	assert.Equal(t, env["USERPROFILE"], "c:\\foo\\c")
+	assert.Equal(t, "c:\\foo\\c", env["USERPROFILE"])
 }
 
 func TestInheritEnvVarsWithAbsentTFConfigFile(t *testing.T) {
diff --git a/bundle/internal/schema/main_test.go b/bundle/internal/schema/main_test.go
index 4eeb41d47..06e89c856 100644
--- a/bundle/internal/schema/main_test.go
+++ b/bundle/internal/schema/main_test.go
@@ -2,7 +2,6 @@ package main
 
 import (
 	"bytes"
-	"fmt"
 	"io"
 	"os"
 	"path"
@@ -80,7 +79,7 @@ func TestRequiredAnnotationsForNewFields(t *testing.T) {
 		},
 	})
 	assert.NoError(t, err)
-	assert.Empty(t, updatedFieldPaths, fmt.Sprintf("Missing JSON-schema descriptions for new config fields in bundle/internal/schema/annotations.yml:\n%s", strings.Join(updatedFieldPaths, "\n")))
+	assert.Empty(t, updatedFieldPaths, "Missing JSON-schema descriptions for new config fields in bundle/internal/schema/annotations.yml:\n%s", strings.Join(updatedFieldPaths, "\n"))
 }
 
 // Checks whether types in annotation files are still present in Config type
diff --git a/bundle/libraries/filer_volume_test.go b/bundle/libraries/filer_volume_test.go
index 7b2f5c5ba..2af54b0cb 100644
--- a/bundle/libraries/filer_volume_test.go
+++ b/bundle/libraries/filer_volume_test.go
@@ -212,12 +212,12 @@ func TestFilerForVolumeWithInvalidVolumePaths(t *testing.T) {
 		bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}})
 
 		_, _, diags := GetFilerForLibraries(context.Background(), b)
-		require.Equal(t, diags, diag.Diagnostics{{
+		require.Equal(t, diag.Diagnostics{{
 			Severity:  diag.Error,
 			Summary:   fmt.Sprintf("expected UC volume path to be in the format /Volumes/<catalog>/<schema>/<volume>/..., got %s", p),
 			Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}},
 			Paths:     []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")},
-		}})
+		}}, diags)
 	}
 }
 
diff --git a/bundle/libraries/helpers_test.go b/bundle/libraries/helpers_test.go
index 9d7e12ee5..754aa8f95 100644
--- a/bundle/libraries/helpers_test.go
+++ b/bundle/libraries/helpers_test.go
@@ -12,25 +12,25 @@ func TestLibraryPath(t *testing.T) {
 
 	p, err := libraryPath(&compute.Library{Whl: path})
 	assert.Equal(t, path, p)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	p, err = libraryPath(&compute.Library{Jar: path})
 	assert.Equal(t, path, p)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	p, err = libraryPath(&compute.Library{Egg: path})
 	assert.Equal(t, path, p)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	p, err = libraryPath(&compute.Library{Requirements: path})
 	assert.Equal(t, path, p)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	p, err = libraryPath(&compute.Library{})
 	assert.Equal(t, "", p)
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 
 	p, err = libraryPath(&compute.Library{Pypi: &compute.PythonPyPiLibrary{Package: "pypipackage"}})
 	assert.Equal(t, "", p)
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 }
diff --git a/bundle/permissions/filter_test.go b/bundle/permissions/filter_test.go
index e6e5a3799..ef7167d75 100644
--- a/bundle/permissions/filter_test.go
+++ b/bundle/permissions/filter_test.go
@@ -99,32 +99,32 @@ func TestFilterCurrentUser(t *testing.T) {
 	assert.NoError(t, diags.Error())
 
 	// Assert current user is filtered out.
-	assert.Equal(t, 2, len(b.Config.Resources.Jobs["job1"].Permissions))
+	assert.Len(t, b.Config.Resources.Jobs["job1"].Permissions, 2)
 	assert.Contains(t, b.Config.Resources.Jobs["job1"].Permissions, robot)
 	assert.Contains(t, b.Config.Resources.Jobs["job1"].Permissions, bob)
 
-	assert.Equal(t, 2, len(b.Config.Resources.Jobs["job2"].Permissions))
+	assert.Len(t, b.Config.Resources.Jobs["job2"].Permissions, 2)
 	assert.Contains(t, b.Config.Resources.Jobs["job2"].Permissions, robot)
 	assert.Contains(t, b.Config.Resources.Jobs["job2"].Permissions, bob)
 
-	assert.Equal(t, 2, len(b.Config.Resources.Pipelines["pipeline1"].Permissions))
+	assert.Len(t, b.Config.Resources.Pipelines["pipeline1"].Permissions, 2)
 	assert.Contains(t, b.Config.Resources.Pipelines["pipeline1"].Permissions, robot)
 	assert.Contains(t, b.Config.Resources.Pipelines["pipeline1"].Permissions, bob)
 
-	assert.Equal(t, 2, len(b.Config.Resources.Experiments["experiment1"].Permissions))
+	assert.Len(t, b.Config.Resources.Experiments["experiment1"].Permissions, 2)
 	assert.Contains(t, b.Config.Resources.Experiments["experiment1"].Permissions, robot)
 	assert.Contains(t, b.Config.Resources.Experiments["experiment1"].Permissions, bob)
 
-	assert.Equal(t, 2, len(b.Config.Resources.Models["model1"].Permissions))
+	assert.Len(t, b.Config.Resources.Models["model1"].Permissions, 2)
 	assert.Contains(t, b.Config.Resources.Models["model1"].Permissions, robot)
 	assert.Contains(t, b.Config.Resources.Models["model1"].Permissions, bob)
 
-	assert.Equal(t, 2, len(b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions))
+	assert.Len(t, b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions, 2)
 	assert.Contains(t, b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions, robot)
 	assert.Contains(t, b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions, bob)
 
 	// Assert there's no change to the grant.
-	assert.Equal(t, 1, len(b.Config.Resources.RegisteredModels["registered_model1"].Grants))
+	assert.Len(t, b.Config.Resources.RegisteredModels["registered_model1"].Grants, 1)
 }
 
 func TestFilterCurrentServicePrincipal(t *testing.T) {
@@ -134,32 +134,32 @@ func TestFilterCurrentServicePrincipal(t *testing.T) {
 	assert.NoError(t, diags.Error())
 
 	// Assert current user is filtered out.
-	assert.Equal(t, 2, len(b.Config.Resources.Jobs["job1"].Permissions))
+	assert.Len(t, b.Config.Resources.Jobs["job1"].Permissions, 2)
 	assert.Contains(t, b.Config.Resources.Jobs["job1"].Permissions, alice)
 	assert.Contains(t, b.Config.Resources.Jobs["job1"].Permissions, bob)
 
-	assert.Equal(t, 2, len(b.Config.Resources.Jobs["job2"].Permissions))
+	assert.Len(t, b.Config.Resources.Jobs["job2"].Permissions, 2)
 	assert.Contains(t, b.Config.Resources.Jobs["job2"].Permissions, alice)
 	assert.Contains(t, b.Config.Resources.Jobs["job2"].Permissions, bob)
 
-	assert.Equal(t, 2, len(b.Config.Resources.Pipelines["pipeline1"].Permissions))
+	assert.Len(t, b.Config.Resources.Pipelines["pipeline1"].Permissions, 2)
 	assert.Contains(t, b.Config.Resources.Pipelines["pipeline1"].Permissions, alice)
 	assert.Contains(t, b.Config.Resources.Pipelines["pipeline1"].Permissions, bob)
 
-	assert.Equal(t, 2, len(b.Config.Resources.Experiments["experiment1"].Permissions))
+	assert.Len(t, b.Config.Resources.Experiments["experiment1"].Permissions, 2)
 	assert.Contains(t, b.Config.Resources.Experiments["experiment1"].Permissions, alice)
 	assert.Contains(t, b.Config.Resources.Experiments["experiment1"].Permissions, bob)
 
-	assert.Equal(t, 2, len(b.Config.Resources.Models["model1"].Permissions))
+	assert.Len(t, b.Config.Resources.Models["model1"].Permissions, 2)
 	assert.Contains(t, b.Config.Resources.Models["model1"].Permissions, alice)
 	assert.Contains(t, b.Config.Resources.Models["model1"].Permissions, bob)
 
-	assert.Equal(t, 2, len(b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions))
+	assert.Len(t, b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions, 2)
 	assert.Contains(t, b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions, alice)
 	assert.Contains(t, b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions, bob)
 
 	// Assert there's no change to the grant.
-	assert.Equal(t, 1, len(b.Config.Resources.RegisteredModels["registered_model1"].Grants))
+	assert.Len(t, b.Config.Resources.RegisteredModels["registered_model1"].Grants, 1)
 }
 
 func TestFilterCurrentUserDoesNotErrorWhenNoResources(t *testing.T) {
diff --git a/bundle/permissions/mutator_test.go b/bundle/permissions/mutator_test.go
index 78703e90f..15586e979 100644
--- a/bundle/permissions/mutator_test.go
+++ b/bundle/permissions/mutator_test.go
@@ -164,7 +164,7 @@ func TestAllResourcesExplicitlyDefinedForPermissionsSupport(t *testing.T) {
 
 	for _, resource := range unsupportedResources {
 		_, ok := levelsMap[resource]
-		assert.False(t, ok, fmt.Sprintf("Resource %s is defined in both levelsMap and unsupportedResources", resource))
+		assert.False(t, ok, "Resource %s is defined in both levelsMap and unsupportedResources", resource)
 	}
 
 	for _, resource := range r.AllResources() {
diff --git a/bundle/permissions/permission_diagnostics_test.go b/bundle/permissions/permission_diagnostics_test.go
index 7b0afefa0..6c55ab594 100644
--- a/bundle/permissions/permission_diagnostics_test.go
+++ b/bundle/permissions/permission_diagnostics_test.go
@@ -28,7 +28,7 @@ func TestPermissionDiagnosticsApplyFail(t *testing.T) {
 	})
 
 	diags := permissions.PermissionDiagnostics().Apply(context.Background(), b)
-	require.Equal(t, diags[0].Severity, diag.Warning)
+	require.Equal(t, diag.Warning, diags[0].Severity)
 	require.Contains(t, diags[0].Summary, "permissions section should include testuser@databricks.com or one of their groups with CAN_MANAGE permissions")
 }
 
diff --git a/bundle/run/job_test.go b/bundle/run/job_test.go
index 5d19ca4ff..72aecc887 100644
--- a/bundle/run/job_test.go
+++ b/bundle/run/job_test.go
@@ -54,7 +54,7 @@ func TestConvertPythonParams(t *testing.T) {
 	err = runner.convertPythonParams(opts)
 	require.NoError(t, err)
 	require.Contains(t, opts.Job.notebookParams, "__python_params")
-	require.Equal(t, opts.Job.notebookParams["__python_params"], `["param1","param2","param3"]`)
+	require.Equal(t, `["param1","param2","param3"]`, opts.Job.notebookParams["__python_params"])
 }
 
 func TestJobRunnerCancel(t *testing.T) {
diff --git a/bundle/tests/complex_variables_test.go b/bundle/tests/complex_variables_test.go
index e68823c33..d72b5f157 100644
--- a/bundle/tests/complex_variables_test.go
+++ b/bundle/tests/complex_variables_test.go
@@ -30,7 +30,7 @@ func TestComplexVariables(t *testing.T) {
 	require.Equal(t, "true", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkConf["spark.speculation"])
 	require.Equal(t, "true", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkConf["spark.random"])
 
-	require.Equal(t, 3, len(b.Config.Resources.Jobs["my_job"].Tasks[0].Libraries))
+	require.Len(t, b.Config.Resources.Jobs["my_job"].Tasks[0].Libraries, 3)
 	require.Contains(t, b.Config.Resources.Jobs["my_job"].Tasks[0].Libraries, compute.Library{
 		Jar: "/path/to/jar",
 	})
diff --git a/bundle/tests/environment_git_test.go b/bundle/tests/environment_git_test.go
index d4695c78d..848b972b1 100644
--- a/bundle/tests/environment_git_test.go
+++ b/bundle/tests/environment_git_test.go
@@ -2,7 +2,6 @@ package config_tests
 
 import (
 	"context"
-	"fmt"
 	"strings"
 	"testing"
 
@@ -16,7 +15,7 @@ func TestGitAutoLoadWithEnvironment(t *testing.T) {
 	bundle.Apply(context.Background(), b, mutator.LoadGitDetails())
 	assert.True(t, b.Config.Bundle.Git.Inferred)
 	validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
-	assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL))
+	assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)
 }
 
 func TestGitManuallySetBranchWithEnvironment(t *testing.T) {
@@ -25,5 +24,5 @@ func TestGitManuallySetBranchWithEnvironment(t *testing.T) {
 	assert.False(t, b.Config.Bundle.Git.Inferred)
 	assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
 	validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
-	assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL))
+	assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)
 }
diff --git a/bundle/tests/environment_overrides_test.go b/bundle/tests/environment_overrides_test.go
index 4a1115048..b68b083ff 100644
--- a/bundle/tests/environment_overrides_test.go
+++ b/bundle/tests/environment_overrides_test.go
@@ -21,8 +21,8 @@ func TestEnvironmentOverridesResourcesDev(t *testing.T) {
 	assert.Equal(t, "base job", b.Config.Resources.Jobs["job1"].Name)
 
 	// Base values are preserved in the development environment.
-	assert.Equal(t, true, b.Config.Resources.Pipelines["boolean1"].Photon)
-	assert.Equal(t, false, b.Config.Resources.Pipelines["boolean2"].Photon)
+	assert.True(t, b.Config.Resources.Pipelines["boolean1"].Photon)
+	assert.False(t, b.Config.Resources.Pipelines["boolean2"].Photon)
 }
 
 func TestEnvironmentOverridesResourcesStaging(t *testing.T) {
@@ -30,6 +30,6 @@ func TestEnvironmentOverridesResourcesStaging(t *testing.T) {
 	assert.Equal(t, "staging job", b.Config.Resources.Jobs["job1"].Name)
 
 	// Override values are applied in the staging environment.
-	assert.Equal(t, false, b.Config.Resources.Pipelines["boolean1"].Photon)
-	assert.Equal(t, true, b.Config.Resources.Pipelines["boolean2"].Photon)
+	assert.False(t, b.Config.Resources.Pipelines["boolean1"].Photon)
+	assert.True(t, b.Config.Resources.Pipelines["boolean2"].Photon)
 }
diff --git a/bundle/tests/environments_job_and_pipeline_test.go b/bundle/tests/environments_job_and_pipeline_test.go
index 218d2e470..423b14c07 100644
--- a/bundle/tests/environments_job_and_pipeline_test.go
+++ b/bundle/tests/environments_job_and_pipeline_test.go
@@ -10,11 +10,11 @@ import (
 
 func TestJobAndPipelineDevelopmentWithEnvironment(t *testing.T) {
 	b := loadTarget(t, "./environments_job_and_pipeline", "development")
-	assert.Len(t, b.Config.Resources.Jobs, 0)
+	assert.Empty(t, b.Config.Resources.Jobs)
 	assert.Len(t, b.Config.Resources.Pipelines, 1)
 
 	p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"]
-	assert.Equal(t, b.Config.Bundle.Mode, config.Development)
+	assert.Equal(t, config.Development, b.Config.Bundle.Mode)
 	assert.True(t, p.Development)
 	require.Len(t, p.Libraries, 1)
 	assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path)
@@ -23,7 +23,7 @@ func TestJobAndPipelineDevelopmentWithEnvironment(t *testing.T) {
 
 func TestJobAndPipelineStagingWithEnvironment(t *testing.T) {
 	b := loadTarget(t, "./environments_job_and_pipeline", "staging")
-	assert.Len(t, b.Config.Resources.Jobs, 0)
+	assert.Empty(t, b.Config.Resources.Jobs)
 	assert.Len(t, b.Config.Resources.Pipelines, 1)
 
 	p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"]
diff --git a/bundle/tests/git_test.go b/bundle/tests/git_test.go
index dec6c268a..41293e450 100644
--- a/bundle/tests/git_test.go
+++ b/bundle/tests/git_test.go
@@ -2,7 +2,6 @@ package config_tests
 
 import (
 	"context"
-	"fmt"
 	"strings"
 	"testing"
 
@@ -17,7 +16,7 @@ func TestGitAutoLoad(t *testing.T) {
 	bundle.Apply(context.Background(), b, mutator.LoadGitDetails())
 	assert.True(t, b.Config.Bundle.Git.Inferred)
 	validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
-	assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL))
+	assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)
 }
 
 func TestGitManuallySetBranch(t *testing.T) {
@@ -26,7 +25,7 @@ func TestGitManuallySetBranch(t *testing.T) {
 	assert.False(t, b.Config.Bundle.Git.Inferred)
 	assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
 	validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
-	assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL))
+	assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)
 }
 
 func TestGitBundleBranchValidation(t *testing.T) {
diff --git a/bundle/tests/issue_1828_test.go b/bundle/tests/issue_1828_test.go
index 5f2becce5..31fcfeb8e 100644
--- a/bundle/tests/issue_1828_test.go
+++ b/bundle/tests/issue_1828_test.go
@@ -35,7 +35,7 @@ func TestIssue1828(t *testing.T) {
 	}
 
 	if assert.Contains(t, b.Config.Variables, "float") {
-		assert.Equal(t, 3.14, b.Config.Variables["float"].Default)
+		assert.InDelta(t, 3.14, b.Config.Variables["float"].Default, 0.0001)
 	}
 
 	if assert.Contains(t, b.Config.Variables, "time") {
@@ -43,6 +43,6 @@ func TestIssue1828(t *testing.T) {
 	}
 
 	if assert.Contains(t, b.Config.Variables, "nil") {
-		assert.Equal(t, nil, b.Config.Variables["nil"].Default)
+		assert.Nil(t, b.Config.Variables["nil"].Default)
 	}
 }
diff --git a/bundle/tests/job_and_pipeline_test.go b/bundle/tests/job_and_pipeline_test.go
index 65aa5bdc4..408e3e3ef 100644
--- a/bundle/tests/job_and_pipeline_test.go
+++ b/bundle/tests/job_and_pipeline_test.go
@@ -10,11 +10,11 @@ import (
 
 func TestJobAndPipelineDevelopment(t *testing.T) {
 	b := loadTarget(t, "./job_and_pipeline", "development")
-	assert.Len(t, b.Config.Resources.Jobs, 0)
+	assert.Empty(t, b.Config.Resources.Jobs)
 	assert.Len(t, b.Config.Resources.Pipelines, 1)
 
 	p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"]
-	assert.Equal(t, b.Config.Bundle.Mode, config.Development)
+	assert.Equal(t, config.Development, b.Config.Bundle.Mode)
 	assert.True(t, p.Development)
 	require.Len(t, p.Libraries, 1)
 	assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path)
@@ -23,7 +23,7 @@ func TestJobAndPipelineDevelopment(t *testing.T) {
 
 func TestJobAndPipelineStaging(t *testing.T) {
 	b := loadTarget(t, "./job_and_pipeline", "staging")
-	assert.Len(t, b.Config.Resources.Jobs, 0)
+	assert.Empty(t, b.Config.Resources.Jobs)
 	assert.Len(t, b.Config.Resources.Pipelines, 1)
 
 	p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"]
diff --git a/bundle/tests/job_cluster_key_test.go b/bundle/tests/job_cluster_key_test.go
index 5a8b368e5..6a08da89c 100644
--- a/bundle/tests/job_cluster_key_test.go
+++ b/bundle/tests/job_cluster_key_test.go
@@ -16,13 +16,13 @@ func TestJobClusterKeyNotDefinedTest(t *testing.T) {
 	diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.JobClusterKeyDefined())
 	require.Len(t, diags, 1)
 	require.NoError(t, diags.Error())
-	require.Equal(t, diags[0].Severity, diag.Warning)
-	require.Equal(t, diags[0].Summary, "job_cluster_key key is not defined")
+	require.Equal(t, diag.Warning, diags[0].Severity)
+	require.Equal(t, "job_cluster_key key is not defined", diags[0].Summary)
 }
 
 func TestJobClusterKeyDefinedTest(t *testing.T) {
 	b := loadTarget(t, "./job_cluster_key", "development")
 
 	diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.JobClusterKeyDefined())
-	require.Len(t, diags, 0)
+	require.Empty(t, diags)
 }
diff --git a/bundle/tests/model_serving_endpoint_test.go b/bundle/tests/model_serving_endpoint_test.go
index b8b800863..f779a07e6 100644
--- a/bundle/tests/model_serving_endpoint_test.go
+++ b/bundle/tests/model_serving_endpoint_test.go
@@ -20,7 +20,7 @@ func assertExpected(t *testing.T, p *resources.ModelServingEndpoint) {
 func TestModelServingEndpointDevelopment(t *testing.T) {
 	b := loadTarget(t, "./model_serving_endpoint", "development")
 	assert.Len(t, b.Config.Resources.ModelServingEndpoints, 1)
-	assert.Equal(t, b.Config.Bundle.Mode, config.Development)
+	assert.Equal(t, config.Development, b.Config.Bundle.Mode)
 
 	p := b.Config.Resources.ModelServingEndpoints["my_model_serving_endpoint"]
 	assert.Equal(t, "my-dev-endpoint", p.Name)
diff --git a/bundle/tests/override_job_tasks_test.go b/bundle/tests/override_job_tasks_test.go
index 82da04da2..85463e17c 100644
--- a/bundle/tests/override_job_tasks_test.go
+++ b/bundle/tests/override_job_tasks_test.go
@@ -12,14 +12,14 @@ func TestOverrideTasksDev(t *testing.T) {
 	assert.Len(t, b.Config.Resources.Jobs["foo"].Tasks, 2)
 
 	tasks := b.Config.Resources.Jobs["foo"].Tasks
-	assert.Equal(t, tasks[0].TaskKey, "key1")
-	assert.Equal(t, tasks[0].NewCluster.NodeTypeId, "i3.xlarge")
-	assert.Equal(t, tasks[0].NewCluster.NumWorkers, 1)
-	assert.Equal(t, tasks[0].SparkPythonTask.PythonFile, "./test1.py")
+	assert.Equal(t, "key1", tasks[0].TaskKey)
+	assert.Equal(t, "i3.xlarge", tasks[0].NewCluster.NodeTypeId)
+	assert.Equal(t, 1, tasks[0].NewCluster.NumWorkers)
+	assert.Equal(t, "./test1.py", tasks[0].SparkPythonTask.PythonFile)
 
-	assert.Equal(t, tasks[1].TaskKey, "key2")
-	assert.Equal(t, tasks[1].NewCluster.SparkVersion, "13.3.x-scala2.12")
-	assert.Equal(t, tasks[1].SparkPythonTask.PythonFile, "./test2.py")
+	assert.Equal(t, "key2", tasks[1].TaskKey)
+	assert.Equal(t, "13.3.x-scala2.12", tasks[1].NewCluster.SparkVersion)
+	assert.Equal(t, "./test2.py", tasks[1].SparkPythonTask.PythonFile)
 }
 
 func TestOverrideTasksStaging(t *testing.T) {
@@ -28,12 +28,12 @@ func TestOverrideTasksStaging(t *testing.T) {
 	assert.Len(t, b.Config.Resources.Jobs["foo"].Tasks, 2)
 
 	tasks := b.Config.Resources.Jobs["foo"].Tasks
-	assert.Equal(t, tasks[0].TaskKey, "key1")
-	assert.Equal(t, tasks[0].NewCluster.SparkVersion, "13.3.x-scala2.12")
-	assert.Equal(t, tasks[0].SparkPythonTask.PythonFile, "./test1.py")
+	assert.Equal(t, "key1", tasks[0].TaskKey)
+	assert.Equal(t, "13.3.x-scala2.12", tasks[0].NewCluster.SparkVersion)
+	assert.Equal(t, "./test1.py", tasks[0].SparkPythonTask.PythonFile)
 
-	assert.Equal(t, tasks[1].TaskKey, "key2")
-	assert.Equal(t, tasks[1].NewCluster.NodeTypeId, "i3.2xlarge")
-	assert.Equal(t, tasks[1].NewCluster.NumWorkers, 4)
-	assert.Equal(t, tasks[1].SparkPythonTask.PythonFile, "./test3.py")
+	assert.Equal(t, "key2", tasks[1].TaskKey)
+	assert.Equal(t, "i3.2xlarge", tasks[1].NewCluster.NodeTypeId)
+	assert.Equal(t, 4, tasks[1].NewCluster.NumWorkers)
+	assert.Equal(t, "./test3.py", tasks[1].SparkPythonTask.PythonFile)
 }
diff --git a/bundle/tests/presets_test.go b/bundle/tests/presets_test.go
index 5fcb5d95b..c2cbe497b 100644
--- a/bundle/tests/presets_test.go
+++ b/bundle/tests/presets_test.go
@@ -13,7 +13,7 @@ func TestPresetsDev(t *testing.T) {
 	assert.Equal(t, "myprefix", b.Config.Presets.NamePrefix)
 	assert.Equal(t, config.Paused, b.Config.Presets.TriggerPauseStatus)
 	assert.Equal(t, 10, b.Config.Presets.JobsMaxConcurrentRuns)
-	assert.Equal(t, true, *b.Config.Presets.PipelinesDevelopment)
+	assert.True(t, *b.Config.Presets.PipelinesDevelopment)
 	assert.Equal(t, "true", b.Config.Presets.Tags["dev"])
 	assert.Equal(t, "finance", b.Config.Presets.Tags["team"])
 	assert.Equal(t, "false", b.Config.Presets.Tags["prod"])
@@ -22,7 +22,7 @@ func TestPresetsDev(t *testing.T) {
 func TestPresetsProd(t *testing.T) {
 	b := loadTarget(t, "./presets", "prod")
 
-	assert.Equal(t, false, *b.Config.Presets.PipelinesDevelopment)
+	assert.False(t, *b.Config.Presets.PipelinesDevelopment)
 	assert.Equal(t, "finance", b.Config.Presets.Tags["team"])
 	assert.Equal(t, "true", b.Config.Presets.Tags["prod"])
 }
diff --git a/bundle/tests/python_wheel_test.go b/bundle/tests/python_wheel_test.go
index c982c09d6..06cb05270 100644
--- a/bundle/tests/python_wheel_test.go
+++ b/bundle/tests/python_wheel_test.go
@@ -23,7 +23,7 @@ func TestPythonWheelBuild(t *testing.T) {
 
 	matches, err := filepath.Glob("./python_wheel/python_wheel/my_test_code/dist/my_test_code-*.whl")
 	require.NoError(t, err)
-	require.Equal(t, 1, len(matches))
+	require.Len(t, matches, 1)
 
 	match := libraries.ExpandGlobReferences()
 	diags = bundle.Apply(ctx, b, match)
@@ -39,7 +39,7 @@ func TestPythonWheelBuildAutoDetect(t *testing.T) {
 
 	matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact/dist/my_test_code-*.whl")
 	require.NoError(t, err)
-	require.Equal(t, 1, len(matches))
+	require.Len(t, matches, 1)
 
 	match := libraries.ExpandGlobReferences()
 	diags = bundle.Apply(ctx, b, match)
@@ -55,7 +55,7 @@ func TestPythonWheelBuildAutoDetectWithNotebookTask(t *testing.T) {
 
 	matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact_notebook/dist/my_test_code-*.whl")
 	require.NoError(t, err)
-	require.Equal(t, 1, len(matches))
+	require.Len(t, matches, 1)
 
 	match := libraries.ExpandGlobReferences()
 	diags = bundle.Apply(ctx, b, match)
@@ -108,7 +108,7 @@ func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) {
 
 	matches, err := filepath.Glob("./python_wheel/environment_key/my_test_code/dist/my_test_code-*.whl")
 	require.NoError(t, err)
-	require.Equal(t, 1, len(matches))
+	require.Len(t, matches, 1)
 
 	match := libraries.ExpandGlobReferences()
 	diags = bundle.Apply(ctx, b, match)
@@ -124,7 +124,7 @@ func TestPythonWheelBuildMultiple(t *testing.T) {
 
 	matches, err := filepath.Glob("./python_wheel/python_wheel_multiple/my_test_code/dist/my_test_code*.whl")
 	require.NoError(t, err)
-	require.Equal(t, 2, len(matches))
+	require.Len(t, matches, 2)
 
 	match := libraries.ExpandGlobReferences()
 	diags = bundle.Apply(ctx, b, match)
diff --git a/bundle/tests/quality_monitor_test.go b/bundle/tests/quality_monitor_test.go
index 9b91052f5..e95c7b7c1 100644
--- a/bundle/tests/quality_monitor_test.go
+++ b/bundle/tests/quality_monitor_test.go
@@ -19,7 +19,7 @@ func assertExpectedMonitor(t *testing.T, p *resources.QualityMonitor) {
 func TestMonitorTableNames(t *testing.T) {
 	b := loadTarget(t, "./quality_monitor", "development")
 	assert.Len(t, b.Config.Resources.QualityMonitors, 1)
-	assert.Equal(t, b.Config.Bundle.Mode, config.Development)
+	assert.Equal(t, config.Development, b.Config.Bundle.Mode)
 
 	p := b.Config.Resources.QualityMonitors["my_monitor"]
 	assert.Equal(t, "main.test.dev", p.TableName)
diff --git a/bundle/tests/registered_model_test.go b/bundle/tests/registered_model_test.go
index 008db8bdd..e9d572a3a 100644
--- a/bundle/tests/registered_model_test.go
+++ b/bundle/tests/registered_model_test.go
@@ -19,7 +19,7 @@ func assertExpectedModel(t *testing.T, p *resources.RegisteredModel) {
 func TestRegisteredModelDevelopment(t *testing.T) {
 	b := loadTarget(t, "./registered_model", "development")
 	assert.Len(t, b.Config.Resources.RegisteredModels, 1)
-	assert.Equal(t, b.Config.Bundle.Mode, config.Development)
+	assert.Equal(t, config.Development, b.Config.Bundle.Mode)
 
 	p := b.Config.Resources.RegisteredModels["my_registered_model"]
 	assert.Equal(t, "my-dev-model", p.Name)
diff --git a/bundle/tests/sync_include_exclude_no_matches_test.go b/bundle/tests/sync_include_exclude_no_matches_test.go
index 0192b61e6..c206e7471 100644
--- a/bundle/tests/sync_include_exclude_no_matches_test.go
+++ b/bundle/tests/sync_include_exclude_no_matches_test.go
@@ -20,26 +20,26 @@ func TestSyncIncludeExcludeNoMatchesTest(t *testing.T) {
 	require.Len(t, diags, 3)
 	require.NoError(t, diags.Error())
 
-	require.Equal(t, diags[0].Severity, diag.Warning)
-	require.Equal(t, diags[0].Summary, "Pattern dist does not match any files")
+	require.Equal(t, diag.Warning, diags[0].Severity)
+	require.Equal(t, "Pattern dist does not match any files", diags[0].Summary)
 
 	require.Len(t, diags[0].Paths, 1)
-	require.Equal(t, diags[0].Paths[0].String(), "sync.exclude[0]")
+	require.Equal(t, "sync.exclude[0]", diags[0].Paths[0].String())
 
 	assert.Len(t, diags[0].Locations, 1)
 	require.Equal(t, diags[0].Locations[0].File, filepath.Join("sync", "override", "databricks.yml"))
-	require.Equal(t, diags[0].Locations[0].Line, 17)
-	require.Equal(t, diags[0].Locations[0].Column, 11)
+	require.Equal(t, 17, diags[0].Locations[0].Line)
+	require.Equal(t, 11, diags[0].Locations[0].Column)
 
 	summaries := []string{
 		fmt.Sprintf("Pattern %s does not match any files", filepath.Join("src", "*")),
 		fmt.Sprintf("Pattern %s does not match any files", filepath.Join("tests", "*")),
 	}
 
-	require.Equal(t, diags[1].Severity, diag.Warning)
+	require.Equal(t, diag.Warning, diags[1].Severity)
 	require.Contains(t, summaries, diags[1].Summary)
 
-	require.Equal(t, diags[2].Severity, diag.Warning)
+	require.Equal(t, diag.Warning, diags[2].Severity)
 	require.Contains(t, summaries, diags[2].Summary)
 }
 
@@ -47,7 +47,7 @@ func TestSyncIncludeWithNegate(t *testing.T) {
 	b := loadTarget(t, "./sync/negate", "default")
 
 	diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.ValidateSyncPatterns())
-	require.Len(t, diags, 0)
+	require.Empty(t, diags)
 	require.NoError(t, diags.Error())
 }
 
@@ -58,6 +58,6 @@ func TestSyncIncludeWithNegateNoMatches(t *testing.T) {
 	require.Len(t, diags, 1)
 	require.NoError(t, diags.Error())
 
-	require.Equal(t, diags[0].Severity, diag.Warning)
-	require.Equal(t, diags[0].Summary, "Pattern !*.txt2 does not match any files")
+	require.Equal(t, diag.Warning, diags[0].Severity)
+	require.Equal(t, "Pattern !*.txt2 does not match any files", diags[0].Summary)
 }
diff --git a/bundle/tests/sync_test.go b/bundle/tests/sync_test.go
index 15644b67e..f5a0296a9 100644
--- a/bundle/tests/sync_test.go
+++ b/bundle/tests/sync_test.go
@@ -115,12 +115,12 @@ func TestSyncPathsNoRoot(t *testing.T) {
 	// If set to nil, it won't sync anything.
 	b = loadTarget(t, "./sync/paths_no_root", "nil")
 	assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath)
-	assert.Len(t, b.Config.Sync.Paths, 0)
+	assert.Empty(t, b.Config.Sync.Paths)
 
 	// If set to an empty sequence, it won't sync anything.
 	b = loadTarget(t, "./sync/paths_no_root", "empty")
 	assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath)
-	assert.Len(t, b.Config.Sync.Paths, 0)
+	assert.Empty(t, b.Config.Sync.Paths)
 }
 
 func TestSyncSharedCode(t *testing.T) {
diff --git a/cmd/bundle/generate/dashboard_test.go b/cmd/bundle/generate/dashboard_test.go
index f1161950b..33a463ea0 100644
--- a/cmd/bundle/generate/dashboard_test.go
+++ b/cmd/bundle/generate/dashboard_test.go
@@ -44,7 +44,7 @@ func TestDashboard_ErrorOnLegacyDashboard(t *testing.T) {
 
 	_, diags := d.resolveID(ctx, b)
 	require.Len(t, diags, 1)
-	assert.Equal(t, diags[0].Summary, "dashboard \"legacy dashboard\" is a legacy dashboard")
+	assert.Equal(t, "dashboard \"legacy dashboard\" is a legacy dashboard", diags[0].Summary)
 }
 
 func TestDashboard_ExistingID_Nominal(t *testing.T) {
diff --git a/cmd/bundle/generate/generate_test.go b/cmd/bundle/generate/generate_test.go
index 896b7de51..395d4ebd4 100644
--- a/cmd/bundle/generate/generate_test.go
+++ b/cmd/bundle/generate/generate_test.go
@@ -3,7 +3,6 @@ package generate
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"io/fs"
@@ -302,7 +301,7 @@ func TestGenerateJobCommandOldFileRename(t *testing.T) {
 
 	// Make sure file do not exists after the run
 	_, err = os.Stat(oldFilename)
-	require.True(t, errors.Is(err, fs.ErrNotExist))
+	require.ErrorIs(t, err, fs.ErrNotExist)
 
 	data, err := os.ReadFile(filepath.Join(configDir, "test_job.job.yml"))
 	require.NoError(t, err)
diff --git a/cmd/configure/configure_test.go b/cmd/configure/configure_test.go
index e2f6c1e29..14eb0674a 100644
--- a/cmd/configure/configure_test.go
+++ b/cmd/configure/configure_test.go
@@ -148,9 +148,9 @@ func TestEnvVarsConfigureNoInteractive(t *testing.T) {
 
 	// We should only save host and token for a profile, other env variables should not be saved
 	_, err = defaultSection.GetKey("auth_type")
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 	_, err = defaultSection.GetKey("metadata_service_url")
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 }
 
 func TestEnvVarsConfigureNoArgsNoInteractive(t *testing.T) {
diff --git a/cmd/labs/github/ref_test.go b/cmd/labs/github/ref_test.go
index cc27d1e81..9668cd7ec 100644
--- a/cmd/labs/github/ref_test.go
+++ b/cmd/labs/github/ref_test.go
@@ -7,14 +7,15 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func TestFileFromRef(t *testing.T) {
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/databrickslabs/ucx/main/README.md" {
 			_, err := w.Write([]byte(`abc`))
-			require.NoError(t, err)
+			if !assert.NoError(t, err) {
+				return
+			}
 			return
 		}
 		t.Logf("Requested: %s", r.URL.Path)
@@ -34,7 +35,9 @@ func TestDownloadZipball(t *testing.T) {
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/repos/databrickslabs/ucx/zipball/main" {
 			_, err := w.Write([]byte(`abc`))
-			require.NoError(t, err)
+			if !assert.NoError(t, err) {
+				return
+			}
 			return
 		}
 		t.Logf("Requested: %s", r.URL.Path)
diff --git a/cmd/labs/github/releases_test.go b/cmd/labs/github/releases_test.go
index 9c3d7a959..93ac33aee 100644
--- a/cmd/labs/github/releases_test.go
+++ b/cmd/labs/github/releases_test.go
@@ -7,14 +7,15 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func TestLoadsReleasesForCLI(t *testing.T) {
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/repos/databricks/cli/releases" {
 			_, err := w.Write([]byte(`[{"tag_name": "v1.2.3"}, {"tag_name": "v1.2.2"}]`))
-			require.NoError(t, err)
+			if !assert.NoError(t, err) {
+				return
+			}
 			return
 		}
 		t.Logf("Requested: %s", r.URL.Path)
diff --git a/cmd/labs/github/repositories_test.go b/cmd/labs/github/repositories_test.go
index 412b440bc..29ec2ce03 100644
--- a/cmd/labs/github/repositories_test.go
+++ b/cmd/labs/github/repositories_test.go
@@ -7,14 +7,13 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func TestRepositories(t *testing.T) {
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/users/databrickslabs/repos" {
 			_, err := w.Write([]byte(`[{"name": "x"}]`))
-			require.NoError(t, err)
+			assert.NoError(t, err)
 			return
 		}
 		t.Logf("Requested: %s", r.URL.Path)
@@ -28,5 +27,5 @@ func TestRepositories(t *testing.T) {
 	r := NewRepositoryCache("databrickslabs", t.TempDir())
 	all, err := r.Load(ctx)
 	assert.NoError(t, err)
-	assert.True(t, len(all) > 0)
+	assert.NotEmpty(t, all)
 }
diff --git a/cmd/labs/localcache/jsonfile_test.go b/cmd/labs/localcache/jsonfile_test.go
index 0d852174c..9d42c6179 100644
--- a/cmd/labs/localcache/jsonfile_test.go
+++ b/cmd/labs/localcache/jsonfile_test.go
@@ -22,7 +22,7 @@ func TestCreatesDirectoryIfNeeded(t *testing.T) {
 	}
 	first, err := c.Load(ctx, tick)
 	assert.NoError(t, err)
-	assert.Equal(t, first, int64(1))
+	assert.Equal(t, int64(1), first)
 }
 
 func TestImpossibleToCreateDir(t *testing.T) {
diff --git a/cmd/labs/project/installer_test.go b/cmd/labs/project/installer_test.go
index a69389b31..a01ba864a 100644
--- a/cmd/labs/project/installer_test.go
+++ b/cmd/labs/project/installer_test.go
@@ -26,6 +26,7 @@ import (
 	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/iam"
 	"github.com/databricks/databricks-sdk-go/service/sql"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -169,17 +170,17 @@ func TestInstallerWorksForReleases(t *testing.T) {
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/databrickslabs/blueprint/v0.3.15/labs.yml" {
 			raw, err := os.ReadFile("testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml")
-			require.NoError(t, err)
+			assert.NoError(t, err)
 			_, err = w.Write(raw)
-			require.NoError(t, err)
+			assert.NoError(t, err)
 			return
 		}
 		if r.URL.Path == "/repos/databrickslabs/blueprint/zipball/v0.3.15" {
 			raw, err := zipballFromFolder("testdata/installed-in-home/.databricks/labs/blueprint/lib")
-			require.NoError(t, err)
+			assert.NoError(t, err)
 			w.Header().Add("Content-Type", "application/octet-stream")
 			_, err = w.Write(raw)
-			require.NoError(t, err)
+			assert.NoError(t, err)
 			return
 		}
 		if r.URL.Path == "/api/2.1/clusters/get" {
@@ -376,17 +377,17 @@ func TestUpgraderWorksForReleases(t *testing.T) {
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/databrickslabs/blueprint/v0.4.0/labs.yml" {
 			raw, err := os.ReadFile("testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml")
-			require.NoError(t, err)
+			assert.NoError(t, err)
 			_, err = w.Write(raw)
-			require.NoError(t, err)
+			assert.NoError(t, err)
 			return
 		}
 		if r.URL.Path == "/repos/databrickslabs/blueprint/zipball/v0.4.0" {
 			raw, err := zipballFromFolder("testdata/installed-in-home/.databricks/labs/blueprint/lib")
-			require.NoError(t, err)
+			assert.NoError(t, err)
 			w.Header().Add("Content-Type", "application/octet-stream")
 			_, err = w.Write(raw)
-			require.NoError(t, err)
+			assert.NoError(t, err)
 			return
 		}
 		if r.URL.Path == "/api/2.1/clusters/get" {
diff --git a/integration/bundle/artifacts_test.go b/integration/bundle/artifacts_test.go
index 1b71a1c3d..9dafe2f83 100644
--- a/integration/bundle/artifacts_test.go
+++ b/integration/bundle/artifacts_test.go
@@ -85,13 +85,13 @@ func TestUploadArtifactFileToCorrectRemotePath(t *testing.T) {
 
 	// The remote path attribute on the artifact file should have been set.
 	require.Regexp(t,
-		regexp.MustCompile(path.Join(regexp.QuoteMeta(wsDir), `.internal/test\.whl`)),
+		path.Join(regexp.QuoteMeta(wsDir), `.internal/test\.whl`),
 		b.Config.Artifacts["test"].Files[0].RemotePath,
 	)
 
 	// The task library path should have been updated to the remote path.
 	require.Regexp(t,
-		regexp.MustCompile(path.Join("/Workspace", regexp.QuoteMeta(wsDir), `.internal/test\.whl`)),
+		path.Join("/Workspace", regexp.QuoteMeta(wsDir), `.internal/test\.whl`),
 		b.Config.Resources.Jobs["test"].JobSettings.Tasks[0].Libraries[0].Whl,
 	)
 }
@@ -149,13 +149,13 @@ func TestUploadArtifactFileToCorrectRemotePathWithEnvironments(t *testing.T) {
 
 	// The remote path attribute on the artifact file should have been set.
 	require.Regexp(t,
-		regexp.MustCompile(path.Join(regexp.QuoteMeta(wsDir), `.internal/test\.whl`)),
+		path.Join(regexp.QuoteMeta(wsDir), `.internal/test\.whl`),
 		b.Config.Artifacts["test"].Files[0].RemotePath,
 	)
 
 	// The job environment deps path should have been updated to the remote path.
 	require.Regexp(t,
-		regexp.MustCompile(path.Join("/Workspace", regexp.QuoteMeta(wsDir), `.internal/test\.whl`)),
+		path.Join("/Workspace", regexp.QuoteMeta(wsDir), `.internal/test\.whl`),
 		b.Config.Resources.Jobs["test"].JobSettings.Environments[0].Spec.Dependencies[0],
 	)
 }
@@ -218,13 +218,13 @@ func TestUploadArtifactFileToCorrectRemotePathForVolumes(t *testing.T) {
 
 	// The remote path attribute on the artifact file should have been set.
 	require.Regexp(t,
-		regexp.MustCompile(path.Join(regexp.QuoteMeta(volumePath), `.internal/test\.whl`)),
+		path.Join(regexp.QuoteMeta(volumePath), `.internal/test\.whl`),
 		b.Config.Artifacts["test"].Files[0].RemotePath,
 	)
 
 	// The task library path should have been updated to the remote path.
 	require.Regexp(t,
-		regexp.MustCompile(path.Join(regexp.QuoteMeta(volumePath), `.internal/test\.whl`)),
+		path.Join(regexp.QuoteMeta(volumePath), `.internal/test\.whl`),
 		b.Config.Resources.Jobs["test"].JobSettings.Tasks[0].Libraries[0].Whl,
 	)
 }
diff --git a/integration/bundle/deploy_test.go b/integration/bundle/deploy_test.go
index 0b37e5630..309b82917 100644
--- a/integration/bundle/deploy_test.go
+++ b/integration/bundle/deploy_test.go
@@ -2,7 +2,6 @@ package bundle_test
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -99,7 +98,7 @@ func TestBundleDeployUcSchema(t *testing.T) {
 	// Assert the schema is deleted
 	_, err = w.Schemas.GetByFullName(ctx, strings.Join([]string{catalogName, schemaName}, "."))
 	apiErr := &apierr.APIError{}
-	assert.True(t, errors.As(err, &apiErr))
+	assert.ErrorAs(t, err, &apiErr)
 	assert.Equal(t, "SCHEMA_DOES_NOT_EXIST", apiErr.ErrorCode)
 }
 
diff --git a/integration/bundle/destroy_test.go b/integration/bundle/destroy_test.go
index f18138ce5..b69382a58 100644
--- a/integration/bundle/destroy_test.go
+++ b/integration/bundle/destroy_test.go
@@ -1,7 +1,6 @@
 package bundle_test
 
 import (
-	"errors"
 	"os"
 	"path/filepath"
 	"testing"
@@ -71,11 +70,11 @@ func TestBundleDestroy(t *testing.T) {
 	// Assert snapshot file is deleted
 	entries, err = os.ReadDir(snapshotsDir)
 	require.NoError(t, err)
-	assert.Len(t, entries, 0)
+	assert.Empty(t, entries)
 
 	// Assert bundle deployment path is deleted
 	_, err = w.Workspace.GetStatusByPath(ctx, remoteRoot)
 	apiErr := &apierr.APIError{}
-	assert.True(t, errors.As(err, &apiErr))
+	assert.ErrorAs(t, err, &apiErr)
 	assert.Equal(t, "RESOURCE_DOES_NOT_EXIST", apiErr.ErrorCode)
 }
diff --git a/integration/cmd/fs/ls_test.go b/integration/cmd/fs/ls_test.go
index 58e776d8a..25929fdf3 100644
--- a/integration/cmd/fs/ls_test.go
+++ b/integration/cmd/fs/ls_test.go
@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"io/fs"
 	"path"
-	"regexp"
 	"strings"
 	"testing"
 
@@ -65,11 +64,11 @@ func TestFsLs(t *testing.T) {
 
 			assert.Equal(t, "a", parsedStdout[0]["name"])
 			assert.Equal(t, true, parsedStdout[0]["is_directory"])
-			assert.Equal(t, float64(0), parsedStdout[0]["size"])
+			assert.InDelta(t, float64(0), parsedStdout[0]["size"], 0.0001)
 
 			assert.Equal(t, "bye.txt", parsedStdout[1]["name"])
 			assert.Equal(t, false, parsedStdout[1]["is_directory"])
-			assert.Equal(t, float64(3), parsedStdout[1]["size"])
+			assert.InDelta(t, float64(3), parsedStdout[1]["size"], 0.0001)
 		})
 	}
 }
@@ -99,11 +98,11 @@ func TestFsLsWithAbsolutePaths(t *testing.T) {
 
 			assert.Equal(t, path.Join(tmpDir, "a"), parsedStdout[0]["name"])
 			assert.Equal(t, true, parsedStdout[0]["is_directory"])
-			assert.Equal(t, float64(0), parsedStdout[0]["size"])
+			assert.InDelta(t, float64(0), parsedStdout[0]["size"], 0.0001)
 
 			assert.Equal(t, path.Join(tmpDir, "bye.txt"), parsedStdout[1]["name"])
 			assert.Equal(t, false, parsedStdout[1]["is_directory"])
-			assert.Equal(t, float64(3), parsedStdout[1]["size"])
+			assert.InDelta(t, float64(3), parsedStdout[1]["size"].(float64), 0.0001)
 		})
 	}
 }
@@ -122,7 +121,7 @@ func TestFsLsOnFile(t *testing.T) {
 			setupLsFiles(t, f)
 
 			_, _, err := testcli.RequireErrorRun(t, ctx, "fs", "ls", path.Join(tmpDir, "a", "hello.txt"), "--output=json")
-			assert.Regexp(t, regexp.MustCompile("not a directory: .*/a/hello.txt"), err.Error())
+			assert.Regexp(t, "not a directory: .*/a/hello.txt", err.Error())
 			assert.ErrorAs(t, err, &filer.NotADirectory{})
 		})
 	}
@@ -147,7 +146,7 @@ func TestFsLsOnEmptyDir(t *testing.T) {
 			require.NoError(t, err)
 
 			// assert on ls output
-			assert.Equal(t, 0, len(parsedStdout))
+			assert.Empty(t, parsedStdout)
 		})
 	}
 }
@@ -166,7 +165,7 @@ func TestFsLsForNonexistingDir(t *testing.T) {
 
 			_, _, err := testcli.RequireErrorRun(t, ctx, "fs", "ls", path.Join(tmpDir, "nonexistent"), "--output=json")
 			assert.ErrorIs(t, err, fs.ErrNotExist)
-			assert.Regexp(t, regexp.MustCompile("no such directory: .*/nonexistent"), err.Error())
+			assert.Regexp(t, "no such directory: .*/nonexistent", err.Error())
 		})
 	}
 }
diff --git a/integration/cmd/fs/mkdir_test.go b/integration/cmd/fs/mkdir_test.go
index f332bb526..eff0599a7 100644
--- a/integration/cmd/fs/mkdir_test.go
+++ b/integration/cmd/fs/mkdir_test.go
@@ -34,7 +34,7 @@ func TestFsMkdir(t *testing.T) {
 			info, err := f.Stat(context.Background(), "a")
 			require.NoError(t, err)
 			assert.Equal(t, "a", info.Name())
-			assert.Equal(t, true, info.IsDir())
+			assert.True(t, info.IsDir())
 		})
 	}
 }
@@ -60,19 +60,19 @@ func TestFsMkdirCreatesIntermediateDirectories(t *testing.T) {
 			infoA, err := f.Stat(context.Background(), "a")
 			require.NoError(t, err)
 			assert.Equal(t, "a", infoA.Name())
-			assert.Equal(t, true, infoA.IsDir())
+			assert.True(t, infoA.IsDir())
 
 			// assert directory "b" is created
 			infoB, err := f.Stat(context.Background(), "a/b")
 			require.NoError(t, err)
 			assert.Equal(t, "b", infoB.Name())
-			assert.Equal(t, true, infoB.IsDir())
+			assert.True(t, infoB.IsDir())
 
 			// assert directory "c" is created
 			infoC, err := f.Stat(context.Background(), "a/b/c")
 			require.NoError(t, err)
 			assert.Equal(t, "c", infoC.Name())
-			assert.Equal(t, true, infoC.IsDir())
+			assert.True(t, infoC.IsDir())
 		})
 	}
 }
diff --git a/integration/cmd/sync/sync_test.go b/integration/cmd/sync/sync_test.go
index 984f6ac49..6f58b7e42 100644
--- a/integration/cmd/sync/sync_test.go
+++ b/integration/cmd/sync/sync_test.go
@@ -225,7 +225,7 @@ func (a *syncTest) snapshotContains(files []string) {
 	assert.Equal(a.t, s.RemotePath, a.remoteRoot)
 	for _, filePath := range files {
 		_, ok := s.LastModifiedTimes[filePath]
-		assert.True(a.t, ok, fmt.Sprintf("%s not in snapshot file: %v", filePath, s.LastModifiedTimes))
+		assert.True(a.t, ok, "%s not in snapshot file: %v", filePath, s.LastModifiedTimes)
 	}
 	assert.Equal(a.t, len(files), len(s.LastModifiedTimes))
 }
diff --git a/integration/libs/filer/filer_test.go b/integration/libs/filer/filer_test.go
index 766f9817b..21c839e1b 100644
--- a/integration/libs/filer/filer_test.go
+++ b/integration/libs/filer/filer_test.go
@@ -4,11 +4,9 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"errors"
 	"io"
 	"io/fs"
 	"path"
-	"regexp"
 	"strings"
 	"testing"
 
@@ -106,7 +104,7 @@ func commonFilerRecursiveDeleteTest(t *testing.T, ctx context.Context, f filer.F
 	for _, e := range entriesBeforeDelete {
 		names = append(names, e.Name())
 	}
-	assert.Equal(t, names, []string{"file1", "file2", "subdir1", "subdir2"})
+	assert.Equal(t, []string{"file1", "file2", "subdir1", "subdir2"}, names)
 
 	err = f.Delete(ctx, "dir")
 	assert.ErrorAs(t, err, &filer.DirectoryNotEmptyError{})
@@ -149,13 +147,13 @@ func commonFilerReadWriteTests(t *testing.T, ctx context.Context, f filer.Filer)
 
 	// Write should fail because the intermediate directory doesn't exist.
 	err = f.Write(ctx, "/foo/bar", strings.NewReader(`hello world`))
-	assert.True(t, errors.As(err, &filer.NoSuchDirectoryError{}))
-	assert.True(t, errors.Is(err, fs.ErrNotExist))
+	assert.ErrorAs(t, err, &filer.NoSuchDirectoryError{})
+	assert.ErrorIs(t, err, fs.ErrNotExist)
 
 	// Read should fail because the intermediate directory doesn't yet exist.
 	_, err = f.Read(ctx, "/foo/bar")
-	assert.True(t, errors.As(err, &filer.FileDoesNotExistError{}))
-	assert.True(t, errors.Is(err, fs.ErrNotExist))
+	assert.ErrorAs(t, err, &filer.FileDoesNotExistError{})
+	assert.ErrorIs(t, err, fs.ErrNotExist)
 
 	// Read should fail because the path points to a directory
 	err = f.Mkdir(ctx, "/dir")
@@ -170,8 +168,8 @@ func commonFilerReadWriteTests(t *testing.T, ctx context.Context, f filer.Filer)
 
 	// Write should fail because there is an existing file at the specified path.
 	err = f.Write(ctx, "/foo/bar", strings.NewReader(`hello universe`))
-	assert.True(t, errors.As(err, &filer.FileAlreadyExistsError{}))
-	assert.True(t, errors.Is(err, fs.ErrExist))
+	assert.ErrorAs(t, err, &filer.FileAlreadyExistsError{})
+	assert.ErrorIs(t, err, fs.ErrExist)
 
 	// Write with OverwriteIfExists should succeed.
 	err = f.Write(ctx, "/foo/bar", strings.NewReader(`hello universe`), filer.OverwriteIfExists)
@@ -188,7 +186,7 @@ func commonFilerReadWriteTests(t *testing.T, ctx context.Context, f filer.Filer)
 	require.NoError(t, err)
 	assert.Equal(t, "foo", info.Name())
 	assert.True(t, info.Mode().IsDir())
-	assert.Equal(t, true, info.IsDir())
+	assert.True(t, info.IsDir())
 
 	// Stat on a file should succeed.
 	// Note: size and modification time behave differently between backends.
@@ -196,17 +194,17 @@ func commonFilerReadWriteTests(t *testing.T, ctx context.Context, f filer.Filer)
 	require.NoError(t, err)
 	assert.Equal(t, "bar", info.Name())
 	assert.True(t, info.Mode().IsRegular())
-	assert.Equal(t, false, info.IsDir())
+	assert.False(t, info.IsDir())
 
 	// Delete should fail if the file doesn't exist.
 	err = f.Delete(ctx, "/doesnt_exist")
 	assert.ErrorAs(t, err, &filer.FileDoesNotExistError{})
-	assert.True(t, errors.Is(err, fs.ErrNotExist))
+	assert.ErrorIs(t, err, fs.ErrNotExist)
 
 	// Stat should fail if the file doesn't exist.
 	_, err = f.Stat(ctx, "/doesnt_exist")
 	assert.ErrorAs(t, err, &filer.FileDoesNotExistError{})
-	assert.True(t, errors.Is(err, fs.ErrNotExist))
+	assert.ErrorIs(t, err, fs.ErrNotExist)
 
 	// Delete should succeed for file that does exist.
 	err = f.Delete(ctx, "/foo/bar")
@@ -215,7 +213,7 @@ func commonFilerReadWriteTests(t *testing.T, ctx context.Context, f filer.Filer)
 	// Delete should fail for a non-empty directory.
 	err = f.Delete(ctx, "/foo")
 	assert.ErrorAs(t, err, &filer.DirectoryNotEmptyError{})
-	assert.True(t, errors.Is(err, fs.ErrInvalid))
+	assert.ErrorIs(t, err, fs.ErrInvalid)
 
 	// Delete should succeed for a non-empty directory if the DeleteRecursively flag is set.
 	err = f.Delete(ctx, "/foo", filer.DeleteRecursively)
@@ -224,8 +222,8 @@ func commonFilerReadWriteTests(t *testing.T, ctx context.Context, f filer.Filer)
 	// Delete of the filer root should ALWAYS fail, otherwise subsequent writes would fail.
 	// It is not in the filer's purview to delete its root directory.
 	err = f.Delete(ctx, "/")
-	assert.True(t, errors.As(err, &filer.CannotDeleteRootError{}))
-	assert.True(t, errors.Is(err, fs.ErrInvalid))
+	assert.ErrorAs(t, err, &filer.CannotDeleteRootError{})
+	assert.ErrorIs(t, err, fs.ErrInvalid)
 }
 
 func TestFilerReadWrite(t *testing.T) {
@@ -262,7 +260,7 @@ func commonFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) {
 	// We start with an empty directory.
 	entries, err := f.ReadDir(ctx, ".")
 	require.NoError(t, err)
-	assert.Len(t, entries, 0)
+	assert.Empty(t, entries)
 
 	// Write a file.
 	err = f.Write(ctx, "/hello.txt", strings.NewReader(`hello world`))
@@ -282,8 +280,8 @@ func commonFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) {
 
 	// Expect an error if the path doesn't exist.
 	_, err = f.ReadDir(ctx, "/dir/a/b/c/d/e")
-	assert.True(t, errors.As(err, &filer.NoSuchDirectoryError{}), err)
-	assert.True(t, errors.Is(err, fs.ErrNotExist))
+	assert.ErrorAs(t, err, &filer.NoSuchDirectoryError{}, err)
+	assert.ErrorIs(t, err, fs.ErrNotExist)
 
 	// Expect two entries in the root.
 	entries, err = f.ReadDir(ctx, ".")
@@ -295,7 +293,7 @@ func commonFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) {
 	assert.False(t, entries[1].IsDir())
 	info, err = entries[1].Info()
 	require.NoError(t, err)
-	assert.Greater(t, info.ModTime().Unix(), int64(0))
+	assert.Positive(t, info.ModTime().Unix())
 
 	// Expect two entries in the directory.
 	entries, err = f.ReadDir(ctx, "/dir")
@@ -307,7 +305,7 @@ func commonFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) {
 	assert.False(t, entries[1].IsDir())
 	info, err = entries[1].Info()
 	require.NoError(t, err)
-	assert.Greater(t, info.ModTime().Unix(), int64(0))
+	assert.Positive(t, info.ModTime().Unix())
 
 	// Expect a single entry in the nested path.
 	entries, err = f.ReadDir(ctx, "/dir/a/b")
@@ -325,7 +323,7 @@ func commonFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) {
 	require.NoError(t, err)
 	entries, err = f.ReadDir(ctx, "empty-dir")
 	assert.NoError(t, err)
-	assert.Len(t, entries, 0)
+	assert.Empty(t, entries)
 
 	// Expect one entry for a directory with a file in it
 	err = f.Write(ctx, "dir-with-one-file/my-file.txt", strings.NewReader("abc"), filer.CreateParentDirectories)
@@ -333,7 +331,7 @@ func commonFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) {
 	entries, err = f.ReadDir(ctx, "dir-with-one-file")
 	assert.NoError(t, err)
 	assert.Len(t, entries, 1)
-	assert.Equal(t, entries[0].Name(), "my-file.txt")
+	assert.Equal(t, "my-file.txt", entries[0].Name())
 	assert.False(t, entries[0].IsDir())
 }
 
@@ -459,7 +457,7 @@ func TestFilerWorkspaceNotebook(t *testing.T) {
 			// Assert uploading a second time fails due to overwrite mode missing
 			err = f.Write(ctx, tc.name, strings.NewReader(tc.content2))
 			require.ErrorIs(t, err, fs.ErrExist)
-			assert.Regexp(t, regexp.MustCompile(`file already exists: .*/`+tc.nameWithoutExt+`$`), err.Error())
+			assert.Regexp(t, `file already exists: .*/`+tc.nameWithoutExt+`$`, err.Error())
 
 			// Try uploading the notebook again with overwrite flag. This time it should succeed.
 			err = f.Write(ctx, tc.name, strings.NewReader(tc.content2), filer.OverwriteIfExists)
diff --git a/libs/auth/cache/file_test.go b/libs/auth/cache/file_test.go
index 3e4aae36f..54964bed3 100644
--- a/libs/auth/cache/file_test.go
+++ b/libs/auth/cache/file_test.go
@@ -42,7 +42,7 @@ func TestStoreAndLookup(t *testing.T) {
 	tok, err := l.Lookup("x")
 	require.NoError(t, err)
 	assert.Equal(t, "abc", tok.AccessToken)
-	assert.Equal(t, 2, len(l.Tokens))
+	assert.Len(t, l.Tokens, 2)
 
 	_, err = l.Lookup("z")
 	assert.Equal(t, ErrNotConfigured, err)
diff --git a/libs/databrickscfg/ops_test.go b/libs/databrickscfg/ops_test.go
index 3ea92024c..dd8484fb7 100644
--- a/libs/databrickscfg/ops_test.go
+++ b/libs/databrickscfg/ops_test.go
@@ -216,7 +216,7 @@ func TestSaveToProfile_ClearingPreviousProfile(t *testing.T) {
 
 	dlft, err := file.GetSection("DEFAULT")
 	assert.NoError(t, err)
-	assert.Len(t, dlft.KeysHash(), 0)
+	assert.Empty(t, dlft.KeysHash())
 
 	abc, err := file.GetSection("abc")
 	assert.NoError(t, err)
diff --git a/libs/databrickscfg/profile/file_test.go b/libs/databrickscfg/profile/file_test.go
index 8e5cfefc0..6bcaec4b7 100644
--- a/libs/databrickscfg/profile/file_test.go
+++ b/libs/databrickscfg/profile/file_test.go
@@ -11,10 +11,10 @@ import (
 )
 
 func TestProfileCloud(t *testing.T) {
-	assert.Equal(t, Profile{Host: "https://dbc-XXXXXXXX-YYYY.cloud.databricks.com"}.Cloud(), "AWS")
-	assert.Equal(t, Profile{Host: "https://adb-xxx.y.azuredatabricks.net/"}.Cloud(), "Azure")
-	assert.Equal(t, Profile{Host: "https://workspace.gcp.databricks.com/"}.Cloud(), "GCP")
-	assert.Equal(t, Profile{Host: "https://some.invalid.host.com/"}.Cloud(), "AWS")
+	assert.Equal(t, "AWS", Profile{Host: "https://dbc-XXXXXXXX-YYYY.cloud.databricks.com"}.Cloud())
+	assert.Equal(t, "Azure", Profile{Host: "https://adb-xxx.y.azuredatabricks.net/"}.Cloud())
+	assert.Equal(t, "GCP", Profile{Host: "https://workspace.gcp.databricks.com/"}.Cloud())
+	assert.Equal(t, "AWS", Profile{Host: "https://some.invalid.host.com/"}.Cloud())
 }
 
 func TestProfilesSearchCaseInsensitive(t *testing.T) {
diff --git a/libs/errs/aggregate_test.go b/libs/errs/aggregate_test.go
index a307e956f..1af57e099 100644
--- a/libs/errs/aggregate_test.go
+++ b/libs/errs/aggregate_test.go
@@ -1,7 +1,6 @@
 package errs
 
 import (
-	"errors"
 	"fmt"
 	"testing"
 
@@ -14,13 +13,13 @@ func TestFromManyErrors(t *testing.T) {
 	e3 := fmt.Errorf("Error 3")
 	err := FromMany(e1, e2, e3)
 
-	assert.True(t, errors.Is(err, e1))
-	assert.True(t, errors.Is(err, e2))
-	assert.True(t, errors.Is(err, e3))
+	assert.ErrorIs(t, err, e1)
+	assert.ErrorIs(t, err, e2)
+	assert.ErrorIs(t, err, e3)
 
-	assert.Equal(t, err.Error(), `Error 1
+	assert.Equal(t, `Error 1
 Error 2
-Error 3`)
+Error 3`, err.Error())
 }
 
 func TestFromManyErrorsWihtNil(t *testing.T) {
@@ -29,9 +28,9 @@ func TestFromManyErrorsWihtNil(t *testing.T) {
 	e3 := fmt.Errorf("Error 3")
 	err := FromMany(e1, e2, e3)
 
-	assert.True(t, errors.Is(err, e1))
-	assert.True(t, errors.Is(err, e3))
+	assert.ErrorIs(t, err, e1)
+	assert.ErrorIs(t, err, e3)
 
-	assert.Equal(t, err.Error(), `Error 1
-Error 3`)
+	assert.Equal(t, `Error 1
+Error 3`, err.Error())
 }
diff --git a/libs/filer/completer/completer_test.go b/libs/filer/completer/completer_test.go
index d284447b9..865d34c2f 100644
--- a/libs/filer/completer/completer_test.go
+++ b/libs/filer/completer/completer_test.go
@@ -37,7 +37,7 @@ func TestFilerCompleterSetsPrefix(t *testing.T) {
 
 	assert.Equal(t, []string{"dbfs:/dir/dirA/", "dbfs:/dir/dirB/"}, completions)
 	assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 }
 
 func TestFilerCompleterReturnsNestedDirs(t *testing.T) {
@@ -46,7 +46,7 @@ func TestFilerCompleterReturnsNestedDirs(t *testing.T) {
 
 	assert.Equal(t, []string{"dir/dirA/", "dir/dirB/"}, completions)
 	assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 }
 
 func TestFilerCompleterReturnsAdjacentDirs(t *testing.T) {
@@ -55,7 +55,7 @@ func TestFilerCompleterReturnsAdjacentDirs(t *testing.T) {
 
 	assert.Equal(t, []string{"dir/dirA/", "dir/dirB/"}, completions)
 	assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 }
 
 func TestFilerCompleterReturnsNestedDirsAndFiles(t *testing.T) {
@@ -64,7 +64,7 @@ func TestFilerCompleterReturnsNestedDirsAndFiles(t *testing.T) {
 
 	assert.Equal(t, []string{"dir/dirA/", "dir/dirB/", "dir/fileA"}, completions)
 	assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 }
 
 func TestFilerCompleterAddsDbfsPath(t *testing.T) {
@@ -78,7 +78,7 @@ func TestFilerCompleterAddsDbfsPath(t *testing.T) {
 
 	assert.Equal(t, []string{"dir/dirA/", "dir/dirB/", "dbfs:/"}, completions)
 	assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 }
 
 func TestFilerCompleterWindowsSeparator(t *testing.T) {
@@ -92,7 +92,7 @@ func TestFilerCompleterWindowsSeparator(t *testing.T) {
 
 	assert.Equal(t, []string{"dir\\dirA\\", "dir\\dirB\\", "dbfs:/"}, completions)
 	assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 }
 
 func TestFilerCompleterNoCompletions(t *testing.T) {
diff --git a/libs/filer/fs_test.go b/libs/filer/fs_test.go
index 849cf6f7c..08d7a9428 100644
--- a/libs/filer/fs_test.go
+++ b/libs/filer/fs_test.go
@@ -63,7 +63,7 @@ func TestFsOpenFile(t *testing.T) {
 	assert.Equal(t, "fileA", info.Name())
 	assert.Equal(t, int64(3), info.Size())
 	assert.Equal(t, fs.FileMode(0), info.Mode())
-	assert.Equal(t, false, info.IsDir())
+	assert.False(t, info.IsDir())
 
 	// Read until closed.
 	b := make([]byte, 3)
@@ -91,7 +91,7 @@ func TestFsOpenDir(t *testing.T) {
 	info, err := fakeFile.Stat()
 	require.NoError(t, err)
 	assert.Equal(t, "root", info.Name())
-	assert.Equal(t, true, info.IsDir())
+	assert.True(t, info.IsDir())
 
 	de, ok := fakeFile.(fs.ReadDirFile)
 	require.True(t, ok)
diff --git a/libs/fileset/glob_test.go b/libs/fileset/glob_test.go
index 9eb786db9..0224b2547 100644
--- a/libs/fileset/glob_test.go
+++ b/libs/fileset/glob_test.go
@@ -52,7 +52,7 @@ func TestGlobFileset(t *testing.T) {
 
 	files, err = g.Files()
 	require.NoError(t, err)
-	require.Equal(t, len(files), 0)
+	require.Empty(t, files)
 }
 
 func TestGlobFilesetWithRelativeRoot(t *testing.T) {
diff --git a/libs/flags/json_flag_test.go b/libs/flags/json_flag_test.go
index b31324011..956a3541c 100644
--- a/libs/flags/json_flag_test.go
+++ b/libs/flags/json_flag_test.go
@@ -123,12 +123,12 @@ func TestJsonUnmarshalForRequest(t *testing.T) {
 	assert.Equal(t, "new job", r.NewSettings.Name)
 	assert.Equal(t, 0, r.NewSettings.TimeoutSeconds)
 	assert.Equal(t, 1, r.NewSettings.MaxConcurrentRuns)
-	assert.Equal(t, 1, len(r.NewSettings.Tasks))
+	assert.Len(t, r.NewSettings.Tasks, 1)
 	assert.Equal(t, "new task", r.NewSettings.Tasks[0].TaskKey)
 	assert.Equal(t, 0, r.NewSettings.Tasks[0].TimeoutSeconds)
 	assert.Equal(t, 0, r.NewSettings.Tasks[0].MaxRetries)
 	assert.Equal(t, 0, r.NewSettings.Tasks[0].MinRetryIntervalMillis)
-	assert.Equal(t, true, r.NewSettings.Tasks[0].RetryOnTimeout)
+	assert.True(t, r.NewSettings.Tasks[0].RetryOnTimeout)
 }
 
 const incorrectJsonData = `{
@@ -280,8 +280,8 @@ func TestJsonUnmarshalForRequestWithForceSendFields(t *testing.T) {
 	assert.NoError(t, diags.Error())
 	assert.Empty(t, diags)
 
-	assert.Equal(t, false, r.NewSettings.NotificationSettings.NoAlertForSkippedRuns)
-	assert.Equal(t, false, r.NewSettings.NotificationSettings.NoAlertForCanceledRuns)
+	assert.False(t, r.NewSettings.NotificationSettings.NoAlertForSkippedRuns)
+	assert.False(t, r.NewSettings.NotificationSettings.NoAlertForCanceledRuns)
 	assert.NotContains(t, r.NewSettings.NotificationSettings.ForceSendFields, "NoAlertForSkippedRuns")
 	assert.Contains(t, r.NewSettings.NotificationSettings.ForceSendFields, "NoAlertForCanceledRuns")
 }
diff --git a/libs/folders/folders_test.go b/libs/folders/folders_test.go
index 17afc4022..d2afc4f2d 100644
--- a/libs/folders/folders_test.go
+++ b/libs/folders/folders_test.go
@@ -33,6 +33,6 @@ func TestFindDirWithLeaf(t *testing.T) {
 	{
 		out, err := FindDirWithLeaf(root, "this-leaf-doesnt-exist-anywhere")
 		assert.ErrorIs(t, err, os.ErrNotExist)
-		assert.Equal(t, out, "")
+		assert.Equal(t, "", out)
 	}
 }
diff --git a/libs/jsonschema/utils_test.go b/libs/jsonschema/utils_test.go
index 89200dae3..954c723d3 100644
--- a/libs/jsonschema/utils_test.go
+++ b/libs/jsonschema/utils_test.go
@@ -96,7 +96,7 @@ func TestTemplateFromString(t *testing.T) {
 	v, err = fromString("1.1", NumberType)
 	assert.NoError(t, err)
 	// Floating point conversions are not perfect
-	assert.True(t, (v.(float64)-1.1) < 0.000001)
+	assert.Less(t, (v.(float64) - 1.1), 0.000001)
 
 	v, err = fromString("12345", IntegerType)
 	assert.NoError(t, err)
@@ -104,7 +104,7 @@ func TestTemplateFromString(t *testing.T) {
 
 	v, err = fromString("123", NumberType)
 	assert.NoError(t, err)
-	assert.Equal(t, float64(123), v)
+	assert.InDelta(t, float64(123), v.(float64), 0.0001)
 
 	_, err = fromString("qrt", ArrayType)
 	assert.EqualError(t, err, "cannot parse string as object of type array. Value of string: \"qrt\"")
diff --git a/libs/notebook/detect_test.go b/libs/notebook/detect_test.go
index 4ede7bf9b..49a67d2d3 100644
--- a/libs/notebook/detect_test.go
+++ b/libs/notebook/detect_test.go
@@ -1,7 +1,6 @@
 package notebook
 
 import (
-	"errors"
 	"io/fs"
 	"os"
 	"path/filepath"
@@ -53,7 +52,7 @@ func TestDetectCallsDetectJupyter(t *testing.T) {
 
 func TestDetectUnknownExtension(t *testing.T) {
 	_, _, err := Detect("./testdata/doesntexist.foobar")
-	assert.True(t, errors.Is(err, fs.ErrNotExist))
+	assert.ErrorIs(t, err, fs.ErrNotExist)
 
 	nb, _, err := Detect("./testdata/unknown_extension.foobar")
 	require.NoError(t, err)
@@ -62,7 +61,7 @@ func TestDetectUnknownExtension(t *testing.T) {
 
 func TestDetectNoExtension(t *testing.T) {
 	_, _, err := Detect("./testdata/doesntexist")
-	assert.True(t, errors.Is(err, fs.ErrNotExist))
+	assert.ErrorIs(t, err, fs.ErrNotExist)
 
 	nb, _, err := Detect("./testdata/no_extension")
 	require.NoError(t, err)
diff --git a/libs/process/background_test.go b/libs/process/background_test.go
index 2e47e814b..7843375cf 100644
--- a/libs/process/background_test.go
+++ b/libs/process/background_test.go
@@ -95,7 +95,7 @@ func TestBackgroundNoStdin(t *testing.T) {
 func TestBackgroundFails(t *testing.T) {
 	ctx := context.Background()
 	_, err := Background(ctx, []string{"ls", "/dev/null/x"})
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 }
 
 func TestBackgroundFailsOnOption(t *testing.T) {
diff --git a/libs/process/forwarded_test.go b/libs/process/forwarded_test.go
index ddb79818f..71f0a6a63 100644
--- a/libs/process/forwarded_test.go
+++ b/libs/process/forwarded_test.go
@@ -27,7 +27,7 @@ func TestForwardedFails(t *testing.T) {
 	err := Forwarded(ctx, []string{
 		"_non_existent_",
 	}, strings.NewReader("abc\n"), &buf, &buf)
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 }
 
 func TestForwardedFailsOnStdinPipe(t *testing.T) {
@@ -39,5 +39,5 @@ func TestForwardedFailsOnStdinPipe(t *testing.T) {
 		c.Stdin = strings.NewReader("x")
 		return nil
 	})
-	assert.NotNil(t, err)
+	assert.Error(t, err)
 }
diff --git a/libs/process/opts_test.go b/libs/process/opts_test.go
index 3a819fbb9..8b5d51928 100644
--- a/libs/process/opts_test.go
+++ b/libs/process/opts_test.go
@@ -41,7 +41,7 @@ func TestWorksWithLibsEnv(t *testing.T) {
 	vars := cmd.Environ()
 	sort.Strings(vars)
 
-	assert.True(t, len(vars) >= 2)
+	assert.GreaterOrEqual(t, len(vars), 2)
 	assert.Equal(t, "CCC=DDD", vars[0])
 	assert.Equal(t, "EEE=FFF", vars[1])
 }
diff --git a/libs/python/interpreters_unix_test.go b/libs/python/interpreters_unix_test.go
index 8471644a1..57adc9279 100644
--- a/libs/python/interpreters_unix_test.go
+++ b/libs/python/interpreters_unix_test.go
@@ -18,7 +18,7 @@ func TestAtLeastOnePythonInstalled(t *testing.T) {
 	assert.NoError(t, err)
 	a := all.Latest()
 	t.Logf("latest is: %s", a)
-	assert.True(t, len(all) > 0)
+	assert.NotEmpty(t, all)
 }
 
 func TestNoInterpretersFound(t *testing.T) {
diff --git a/libs/sync/snapshot_test.go b/libs/sync/snapshot_test.go
index eef526e58..4ba3874ae 100644
--- a/libs/sync/snapshot_test.go
+++ b/libs/sync/snapshot_test.go
@@ -51,7 +51,7 @@ func TestDiff(t *testing.T) {
 	assert.NoError(t, err)
 	change, err := state.diff(ctx, files)
 	assert.NoError(t, err)
-	assert.Len(t, change.delete, 0)
+	assert.Empty(t, change.delete)
 	assert.Len(t, change.put, 2)
 	assert.Contains(t, change.put, "hello.txt")
 	assert.Contains(t, change.put, "world.txt")
@@ -67,7 +67,7 @@ func TestDiff(t *testing.T) {
 	change, err = state.diff(ctx, files)
 	assert.NoError(t, err)
 
-	assert.Len(t, change.delete, 0)
+	assert.Empty(t, change.delete)
 	assert.Len(t, change.put, 1)
 	assert.Contains(t, change.put, "world.txt")
 	assertKeysOfMap(t, state.LastModifiedTimes, []string{"hello.txt", "world.txt"})
@@ -82,7 +82,7 @@ func TestDiff(t *testing.T) {
 	change, err = state.diff(ctx, files)
 	assert.NoError(t, err)
 	assert.Len(t, change.delete, 1)
-	assert.Len(t, change.put, 0)
+	assert.Empty(t, change.put)
 	assert.Contains(t, change.delete, "hello.txt")
 	assertKeysOfMap(t, state.LastModifiedTimes, []string{"world.txt"})
 	assert.Equal(t, map[string]string{"world.txt": "world.txt"}, state.LocalToRemoteNames)
@@ -145,8 +145,8 @@ func TestFolderDiff(t *testing.T) {
 	assert.NoError(t, err)
 	change, err := state.diff(ctx, files)
 	assert.NoError(t, err)
-	assert.Len(t, change.delete, 0)
-	assert.Len(t, change.rmdir, 0)
+	assert.Empty(t, change.delete)
+	assert.Empty(t, change.rmdir)
 	assert.Len(t, change.mkdir, 1)
 	assert.Len(t, change.put, 1)
 	assert.Contains(t, change.mkdir, "foo")
@@ -159,8 +159,8 @@ func TestFolderDiff(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Len(t, change.delete, 1)
 	assert.Len(t, change.rmdir, 1)
-	assert.Len(t, change.mkdir, 0)
-	assert.Len(t, change.put, 0)
+	assert.Empty(t, change.mkdir)
+	assert.Empty(t, change.put)
 	assert.Contains(t, change.delete, "foo/bar")
 	assert.Contains(t, change.rmdir, "foo")
 }
@@ -189,7 +189,7 @@ func TestPythonNotebookDiff(t *testing.T) {
 	foo.Overwrite(t, "# Databricks notebook source\nprint(\"abc\")")
 	change, err := state.diff(ctx, files)
 	assert.NoError(t, err)
-	assert.Len(t, change.delete, 0)
+	assert.Empty(t, change.delete)
 	assert.Len(t, change.put, 1)
 	assert.Contains(t, change.put, "foo.py")
 	assertKeysOfMap(t, state.LastModifiedTimes, []string{"foo.py"})
@@ -233,9 +233,9 @@ func TestPythonNotebookDiff(t *testing.T) {
 	change, err = state.diff(ctx, files)
 	assert.NoError(t, err)
 	assert.Len(t, change.delete, 1)
-	assert.Len(t, change.put, 0)
+	assert.Empty(t, change.put)
 	assert.Contains(t, change.delete, "foo")
-	assert.Len(t, state.LastModifiedTimes, 0)
+	assert.Empty(t, state.LastModifiedTimes)
 	assert.Equal(t, map[string]string{}, state.LocalToRemoteNames)
 	assert.Equal(t, map[string]string{}, state.RemoteToLocalNames)
 }
@@ -264,7 +264,7 @@ func TestErrorWhenIdenticalRemoteName(t *testing.T) {
 	assert.NoError(t, err)
 	change, err := state.diff(ctx, files)
 	assert.NoError(t, err)
-	assert.Len(t, change.delete, 0)
+	assert.Empty(t, change.delete)
 	assert.Len(t, change.put, 2)
 	assert.Contains(t, change.put, "foo.py")
 	assert.Contains(t, change.put, "foo")
@@ -300,7 +300,7 @@ func TestNoErrorRenameWithIdenticalRemoteName(t *testing.T) {
 	assert.NoError(t, err)
 	change, err := state.diff(ctx, files)
 	assert.NoError(t, err)
-	assert.Len(t, change.delete, 0)
+	assert.Empty(t, change.delete)
 	assert.Len(t, change.put, 1)
 	assert.Contains(t, change.put, "foo.py")
 
diff --git a/libs/sync/sync_test.go b/libs/sync/sync_test.go
index 6168dc217..f30431770 100644
--- a/libs/sync/sync_test.go
+++ b/libs/sync/sync_test.go
@@ -59,7 +59,7 @@ func TestGetFileSet(t *testing.T) {
 
 	fileList, err := s.GetFileList(ctx)
 	require.NoError(t, err)
-	require.Equal(t, len(fileList), 10)
+	require.Len(t, fileList, 10)
 
 	inc, err = fileset.NewGlobSet(root, []string{})
 	require.NoError(t, err)
@@ -77,7 +77,7 @@ func TestGetFileSet(t *testing.T) {
 
 	fileList, err = s.GetFileList(ctx)
 	require.NoError(t, err)
-	require.Equal(t, len(fileList), 2)
+	require.Len(t, fileList, 2)
 
 	inc, err = fileset.NewGlobSet(root, []string{"./.databricks/*.go"})
 	require.NoError(t, err)
@@ -95,7 +95,7 @@ func TestGetFileSet(t *testing.T) {
 
 	fileList, err = s.GetFileList(ctx)
 	require.NoError(t, err)
-	require.Equal(t, len(fileList), 11)
+	require.Len(t, fileList, 11)
 }
 
 func TestRecursiveExclude(t *testing.T) {
@@ -125,7 +125,7 @@ func TestRecursiveExclude(t *testing.T) {
 
 	fileList, err := s.GetFileList(ctx)
 	require.NoError(t, err)
-	require.Equal(t, len(fileList), 7)
+	require.Len(t, fileList, 7)
 }
 
 func TestNegateExclude(t *testing.T) {
@@ -155,6 +155,6 @@ func TestNegateExclude(t *testing.T) {
 
 	fileList, err := s.GetFileList(ctx)
 	require.NoError(t, err)
-	require.Equal(t, len(fileList), 1)
-	require.Equal(t, fileList[0].Relative, "test/sub1/sub2/h.txt")
+	require.Len(t, fileList, 1)
+	require.Equal(t, "test/sub1/sub2/h.txt", fileList[0].Relative)
 }
diff --git a/libs/template/config_test.go b/libs/template/config_test.go
index a855019b6..515a0b9f5 100644
--- a/libs/template/config_test.go
+++ b/libs/template/config_test.go
@@ -24,7 +24,7 @@ func TestTemplateConfigAssignValuesFromFile(t *testing.T) {
 	err = c.assignValuesFromFile(filepath.Join(testDir, "config.json"))
 	if assert.NoError(t, err) {
 		assert.Equal(t, int64(1), c.values["int_val"])
-		assert.Equal(t, float64(2), c.values["float_val"])
+		assert.InDelta(t, float64(2), c.values["float_val"].(float64), 0.0001)
 		assert.Equal(t, true, c.values["bool_val"])
 		assert.Equal(t, "hello", c.values["string_val"])
 	}
@@ -44,7 +44,7 @@ func TestTemplateConfigAssignValuesFromFileDoesNotOverwriteExistingConfigs(t *te
 	err = c.assignValuesFromFile(filepath.Join(testDir, "config.json"))
 	if assert.NoError(t, err) {
 		assert.Equal(t, int64(1), c.values["int_val"])
-		assert.Equal(t, float64(2), c.values["float_val"])
+		assert.InDelta(t, float64(2), c.values["float_val"].(float64), 0.0001)
 		assert.Equal(t, true, c.values["bool_val"])
 		assert.Equal(t, "this-is-not-overwritten", c.values["string_val"])
 	}
@@ -89,7 +89,7 @@ func TestTemplateConfigAssignValuesFromDefaultValues(t *testing.T) {
 	err = c.assignDefaultValues(r)
 	if assert.NoError(t, err) {
 		assert.Equal(t, int64(123), c.values["int_val"])
-		assert.Equal(t, float64(123), c.values["float_val"])
+		assert.InDelta(t, float64(123), c.values["float_val"].(float64), 0.0001)
 		assert.Equal(t, true, c.values["bool_val"])
 		assert.Equal(t, "hello", c.values["string_val"])
 	}
@@ -110,7 +110,7 @@ func TestTemplateConfigAssignValuesFromTemplatedDefaultValues(t *testing.T) {
 	err = c.assignDefaultValues(r)
 	if assert.NoError(t, err) {
 		assert.Equal(t, int64(123), c.values["int_val"])
-		assert.Equal(t, float64(123), c.values["float_val"])
+		assert.InDelta(t, float64(123), c.values["float_val"].(float64), 0.0001)
 		assert.Equal(t, true, c.values["bool_val"])
 		assert.Equal(t, "world", c.values["string_val"])
 	}
diff --git a/libs/template/helpers_test.go b/libs/template/helpers_test.go
index d98f40b24..f8bc1f3da 100644
--- a/libs/template/helpers_test.go
+++ b/libs/template/helpers_test.go
@@ -86,7 +86,7 @@ func TestTemplateRandIntFunction(t *testing.T) {
 	assert.Len(t, r.files, 1)
 	randInt, err := strconv.Atoi(strings.TrimSpace(string(r.files[0].(*inMemoryFile).content)))
 	assert.Less(t, randInt, 10)
-	assert.Empty(t, err)
+	assert.NoError(t, err)
 }
 
 func TestTemplateUuidFunction(t *testing.T) {
diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go
index eeb308732..2c14009ff 100644
--- a/libs/template/renderer_test.go
+++ b/libs/template/renderer_test.go
@@ -434,7 +434,7 @@ func TestRendererSkipAllFilesInCurrentDirectory(t *testing.T) {
 	entries, err := os.ReadDir(tmpDir)
 	require.NoError(t, err)
 	// Assert none of the files are persisted to disk, because of {{skip "*"}}
-	assert.Len(t, entries, 0)
+	assert.Empty(t, entries)
 }
 
 func TestRendererSkipPatternsAreRelativeToFileDirectory(t *testing.T) {
@@ -588,8 +588,8 @@ func TestRendererNonTemplatesAreCreatedAsCopyFiles(t *testing.T) {
 	assert.NoError(t, err)
 
 	assert.Len(t, r.files, 1)
-	assert.Equal(t, r.files[0].(*copyFile).srcPath, "not-a-template")
-	assert.Equal(t, r.files[0].RelPath(), "not-a-template")
+	assert.Equal(t, "not-a-template", r.files[0].(*copyFile).srcPath)
+	assert.Equal(t, "not-a-template", r.files[0].RelPath())
 }
 
 func TestRendererFileTreeRendering(t *testing.T) {
@@ -609,7 +609,7 @@ func TestRendererFileTreeRendering(t *testing.T) {
 
 	// Assert in memory representation is created.
 	assert.Len(t, r.files, 1)
-	assert.Equal(t, r.files[0].RelPath(), "my_directory/my_file")
+	assert.Equal(t, "my_directory/my_file", r.files[0].RelPath())
 
 	out, err := filer.NewLocalClient(tmpDir)
 	require.NoError(t, err)
diff --git a/libs/vfs/filer_test.go b/libs/vfs/filer_test.go
index ee1397521..6987c288e 100644
--- a/libs/vfs/filer_test.go
+++ b/libs/vfs/filer_test.go
@@ -2,7 +2,6 @@ package vfs
 
 import (
 	"context"
-	"errors"
 	"io/fs"
 	"os"
 	"path/filepath"
@@ -42,7 +41,7 @@ func TestFilerPath(t *testing.T) {
 
 	// Open non-existent file.
 	_, err = p.Open("doesntexist_test.go")
-	assert.True(t, errors.Is(err, fs.ErrNotExist))
+	assert.ErrorIs(t, err, fs.ErrNotExist)
 
 	// Stat self.
 	s, err = p.Stat("filer_test.go")
@@ -52,7 +51,7 @@ func TestFilerPath(t *testing.T) {
 
 	// Stat non-existent file.
 	_, err = p.Stat("doesntexist_test.go")
-	assert.True(t, errors.Is(err, fs.ErrNotExist))
+	assert.ErrorIs(t, err, fs.ErrNotExist)
 
 	// ReadDir self.
 	entries, err := p.ReadDir(".")
@@ -61,7 +60,7 @@ func TestFilerPath(t *testing.T) {
 
 	// ReadDir non-existent directory.
 	_, err = p.ReadDir("doesntexist")
-	assert.True(t, errors.Is(err, fs.ErrNotExist))
+	assert.ErrorIs(t, err, fs.ErrNotExist)
 
 	// ReadFile self.
 	buf, err = p.ReadFile("filer_test.go")
@@ -70,7 +69,7 @@ func TestFilerPath(t *testing.T) {
 
 	// ReadFile non-existent file.
 	_, err = p.ReadFile("doesntexist_test.go")
-	assert.True(t, errors.Is(err, fs.ErrNotExist))
+	assert.ErrorIs(t, err, fs.ErrNotExist)
 
 	// Parent self.
 	pp := p.Parent()

From ef86d2bcaeff4d0e0f1445bf0e834299a63eadd7 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 2 Jan 2025 12:06:01 +0100
Subject: [PATCH 013/247] Speed up best case for "make test" 12x (#2060)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

On main branch: ‘make test’ takes about 33s
On this branch: ‘make test’ takes about 2.7s

(all measurements are for hot cache)

What’s done (from highest impact to lowest):
- Remove -coverprofile= option - this option was disabling "go test"'s
built-in cache and also it took extra time to calculate the coverage
(extra 21s).
- Exclude ./integration/ folder, there are no unit tests there, but
having it included adds significant time. "go test"'s caching also does
not work there for me, due to TestMain() presence (extra 7.2s).
- Remove dependency on "make lint" - nice to have, but slow to re-check
the whole repo and should already be done by IDE (extra 2.5s).
- Remove dependency on "make vendor" — rarely needed; on CI it is
already executed separately (extra 1.1s).

The coverage option is still available under "make cover". Use "make
showcover" to show it.

I’ve also removed separate "make testonly". If you only want tests, run
"make test". If you want lint+test run "make lint test" etc.

I've also modified the test command, removed unnecessary -short, -v,
--raw-command.
---
 .github/workflows/push.yml |  2 +-
 Makefile                   | 20 ++++++++++++--------
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
index 25bd5f4d6..f7c4d5456 100644
--- a/.github/workflows/push.yml
+++ b/.github/workflows/push.yml
@@ -55,7 +55,7 @@ jobs:
           pip3 install wheel
 
       - name: Run tests
-        run: make testonly
+        run: make test
 
   golangci:
     name: lint
diff --git a/Makefile b/Makefile
index 7dca3b2cf..2e7e34299 100644
--- a/Makefile
+++ b/Makefile
@@ -1,20 +1,24 @@
 default: build
 
-lint: vendor
+PACKAGES=./libs/... ./internal/... ./cmd/... ./bundle/... .
+
+lint:
 	@echo "✓ Linting source code with https://golangci-lint.run/ (with --fix)..."
 	@./lint.sh ./...
 
-lintcheck: vendor
+lintcheck:
 	@echo "✓ Linting source code with https://golangci-lint.run/ ..."
 	@golangci-lint run ./...
 
-test: lint testonly
-
-testonly:
+test:
 	@echo "✓ Running tests ..."
-	@gotestsum --format pkgname-and-test-fails --no-summary=skipped --raw-command go test -v -json -short -coverprofile=coverage.txt ./...
+	@gotestsum --format pkgname-and-test-fails --no-summary=skipped -- ${PACKAGES}
 
-coverage: test
+cover:
+	@echo "✓ Running tests with coverage..."
+	@gotestsum --format pkgname-and-test-fails --no-summary=skipped -- -coverprofile=coverage.txt ${PACKAGES}
+
+showcover:
 	@echo "✓ Opening coverage for unit tests ..."
 	@go tool cover -html=coverage.txt
 
@@ -42,4 +46,4 @@ integration:
 integration-short:
 	$(INTEGRATION) -short
 
-.PHONY: lint lintcheck test testonly coverage build snapshot vendor schema integration integration-short
+.PHONY: lint lintcheck test cover showcover build snapshot vendor schema integration integration-short

From ea8445af9eea3a6416c5c3705f664a5f8b66d085 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 2 Jan 2025 12:18:38 +0100
Subject: [PATCH 014/247] Make "make" output the commands it runs (#2066)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This is useful on CI and locally for debugging and being able to
copy-paste command to tweak the options.

Removed redundant and imprecise messages like "✓ Running tests ...".
---
 Makefile | 27 +++++++++------------------
 1 file changed, 9 insertions(+), 18 deletions(-)

diff --git a/Makefile b/Makefile
index 2e7e34299..b6a62765f 100644
--- a/Makefile
+++ b/Makefile
@@ -3,40 +3,31 @@ default: build
 PACKAGES=./libs/... ./internal/... ./cmd/... ./bundle/... .
 
 lint:
-	@echo "✓ Linting source code with https://golangci-lint.run/ (with --fix)..."
-	@./lint.sh ./...
+	./lint.sh ./...
 
 lintcheck:
-	@echo "✓ Linting source code with https://golangci-lint.run/ ..."
-	@golangci-lint run ./...
+	golangci-lint run ./...
 
 test:
-	@echo "✓ Running tests ..."
-	@gotestsum --format pkgname-and-test-fails --no-summary=skipped -- ${PACKAGES}
+	gotestsum --format pkgname-and-test-fails --no-summary=skipped -- ${PACKAGES}
 
 cover:
-	@echo "✓ Running tests with coverage..."
-	@gotestsum --format pkgname-and-test-fails --no-summary=skipped -- -coverprofile=coverage.txt ${PACKAGES}
+	gotestsum --format pkgname-and-test-fails --no-summary=skipped -- -coverprofile=coverage.txt ${PACKAGES}
 
 showcover:
-	@echo "✓ Opening coverage for unit tests ..."
-	@go tool cover -html=coverage.txt
+	go tool cover -html=coverage.txt
 
 build: vendor
-	@echo "✓ Building source code with go build ..."
-	@go build -mod vendor
+	go build -mod vendor
 
 snapshot:
-	@echo "✓ Building dev snapshot"
-	@go build -o .databricks/databricks
+	go build -o .databricks/databricks
 
 vendor:
-	@echo "✓ Filling vendor folder with library code ..."
-	@go mod vendor
+	go mod vendor
   
 schema:
-	@echo "✓ Generating json-schema ..."
-	@go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json
+	go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json
 
 INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./integration/..." -- -parallel 4 -timeout=2h
 

From 890c57eabeb6252a98bbdfe7fe5a061a6284d48f Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Thu, 2 Jan 2025 16:52:33 +0530
Subject: [PATCH 015/247] Enable debugging integration tests in VS Code (#2053)

## Changes
This PR adds back debugging functionality that was lost during migration
to `internal.Main` as an entry point for integration tests.

The PR that caused the regression:
https://github.com/databricks/cli/pull/2009. Specifically the addition
of internal.Main as the entrypoint for all integration tests.

## Tests
Manually, by trying to debug a test.
---
 integration/internal/acc/debug.go | 4 ++--
 integration/internal/main.go      | 4 +++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/integration/internal/acc/debug.go b/integration/internal/acc/debug.go
index b4939881e..08e385b09 100644
--- a/integration/internal/acc/debug.go
+++ b/integration/internal/acc/debug.go
@@ -11,14 +11,14 @@ import (
 )
 
 // Detects if test is run from "debug test" feature in VS Code.
-func isInDebug() bool {
+func IsInDebug() bool {
 	ex, _ := os.Executable()
 	return strings.HasPrefix(path.Base(ex), "__debug_bin")
 }
 
 // Loads debug environment from ~/.databricks/debug-env.json.
 func loadDebugEnvIfRunFromIDE(t testutil.TestingT, key string) {
-	if !isInDebug() {
+	if !IsInDebug() {
 		return
 	}
 	home, err := os.UserHomeDir()
diff --git a/integration/internal/main.go b/integration/internal/main.go
index 6d69dcf70..6aa2a4c93 100644
--- a/integration/internal/main.go
+++ b/integration/internal/main.go
@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"os"
 	"testing"
+
+	"github.com/databricks/cli/integration/internal/acc"
 )
 
 // Main is the entry point for integration tests.
@@ -11,7 +13,7 @@ import (
 // they are not inadvertently executed when calling `go test ./...`.
 func Main(m *testing.M) {
 	value := os.Getenv("CLOUD_ENV")
-	if value == "" {
+	if value == "" && !acc.IsInDebug() {
 		fmt.Println("CLOUD_ENV is not set, skipping integration tests")
 		return
 	}

From cae21693bbac8b74204a1f36b4c54ec66553de39 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 2 Jan 2025 12:23:48 +0100
Subject: [PATCH 016/247] lint: Raise max issues output (#2067)

By default it stops after 3 issues of a given type, which gives false
impression and also unhelpful if you fixing it with aider.

1000 is almost like unlimited but not unlimited in case there is a bug
in a linter.
---
 .golangci.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.golangci.yaml b/.golangci.yaml
index fd83e0882..3e9a88957 100644
--- a/.golangci.yaml
+++ b/.golangci.yaml
@@ -40,3 +40,5 @@ linters-settings:
       - require-error
 issues:
   exclude-dirs-use-default: false  # recommended by docs https://golangci-lint.run/usage/false-positives/
+  max-issues-per-linter: 1000
+  max-same-issues: 1000

From 509f5aba6ad74765e97e87adf34694967da119f6 Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Thu, 2 Jan 2025 17:09:11 +0530
Subject: [PATCH 017/247] Snooze mlops-stacks integration test (#2063)

## Changes
https://github.com/databricks/mlops-stacks/pull/187 broke mlops-stacks
deployments for non-UC projects. Snoozing the test until upstream is
fixed.

## Tests
The test is skipped on my local machine. CI run will verify it's also
skipped on Github Actions runners.
---
 integration/bundle/init_test.go |  2 ++
 internal/testutil/helpers.go    | 12 ++++++++++++
 2 files changed, 14 insertions(+)

diff --git a/integration/bundle/init_test.go b/integration/bundle/init_test.go
index f5c263ca3..bc81cd3a8 100644
--- a/integration/bundle/init_test.go
+++ b/integration/bundle/init_test.go
@@ -39,6 +39,8 @@ func TestBundleInitErrorOnUnknownFields(t *testing.T) {
 //     make changes that can break the MLOps Stacks DAB. In which case we should
 //     skip this test until the MLOps Stacks DAB is updated to work again.
 func TestBundleInitOnMlopsStacks(t *testing.T) {
+	testutil.SkipUntil(t, "2025-01-09")
+
 	ctx, wt := acc.WorkspaceTest(t)
 	w := wt.W
 
diff --git a/internal/testutil/helpers.go b/internal/testutil/helpers.go
index 019a8e618..44c2c9375 100644
--- a/internal/testutil/helpers.go
+++ b/internal/testutil/helpers.go
@@ -5,6 +5,9 @@ import (
 	"math/rand"
 	"os"
 	"strings"
+	"time"
+
+	"github.com/stretchr/testify/require"
 )
 
 // GetEnvOrSkipTest proceeds with test only with that env variable.
@@ -30,3 +33,12 @@ func RandomName(prefix ...string) string {
 	}
 	return string(b)
 }
+
+func SkipUntil(t TestingT, date string) {
+	deadline, err := time.Parse(time.DateOnly, date)
+	require.NoError(t, err)
+
+	if time.Now().Before(deadline) {
+		t.Skipf("Skipping test until %s. Time right now: %s", deadline.Format(time.DateOnly), time.Now())
+	}
+}

From 7beb0fb8b5bf6f5eb7f11df7b958f713356ca33d Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Thu, 2 Jan 2025 17:23:15 +0530
Subject: [PATCH 018/247] Add validation mutator for volume `artifact_path`
 (#2050)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Changes
This PR:
1. Incrementally improves the error messages shown to the user when the
volume they are referring to in `workspace.artifact_path` does not
exist.
2. Performs this validation in both `bundle validate` and `bundle
deploy` compared to before on just deployments.
3. It runs "fast" validations on `bundle deploy`, which earlier were
only run on `bundle validate`.


## Tests
Unit tests and manually. Also, existing integration tests provide
coverage (`TestUploadArtifactToVolumeNotYetDeployed`,
`TestUploadArtifactFileToVolumeThatDoesNotExist`)

Examples:
```
.venv➜  bundle-playground git:(master) ✗ cli bundle validate
Error: cannot access volume capital.whatever.my_volume: User does not have READ VOLUME on Volume 'capital.whatever.my_volume'.
  at workspace.artifact_path
  in databricks.yml:7:18
```

and

```
.venv➜  bundle-playground git:(master) ✗ cli bundle validate
Error: volume capital.whatever.foobar does not exist
  at workspace.artifact_path
     resources.volumes.foo
  in databricks.yml:7:18
     databricks.yml:12:7

You are using a volume in your artifact_path that is managed by
this bundle but which has not been deployed yet. Please first deploy
the volume using 'bundle deploy' and then switch over to using it in
the artifact_path.
```
---
 bundle/config/validate/fast_validate.go       |  51 ++++
 bundle/config/validate/validate.go            |   9 +-
 .../config/validate/validate_artifact_path.go | 129 +++++++++
 .../validate/validate_artifact_path_test.go   | 244 ++++++++++++++++
 bundle/libraries/filer.go                     |   2 +-
 bundle/libraries/filer_test.go                |   8 -
 bundle/libraries/filer_volume.go              | 124 +--------
 bundle/libraries/filer_volume_test.go         | 262 +-----------------
 bundle/libraries/upload_test.go               |   7 -
 cmd/bundle/deploy.go                          |   2 +
 integration/bundle/artifacts_test.go          |   4 +-
 11 files changed, 444 insertions(+), 398 deletions(-)
 create mode 100644 bundle/config/validate/fast_validate.go
 create mode 100644 bundle/config/validate/validate_artifact_path.go
 create mode 100644 bundle/config/validate/validate_artifact_path_test.go

diff --git a/bundle/config/validate/fast_validate.go b/bundle/config/validate/fast_validate.go
new file mode 100644
index 000000000..47d83036d
--- /dev/null
+++ b/bundle/config/validate/fast_validate.go
@@ -0,0 +1,51 @@
+package validate
+
+import (
+	"context"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+)
+
+// FastValidate runs a subset of fast validation checks. This is a subset of the full
+// suite of validation mutators that satisfy ANY ONE of the following criteria:
+//
+// 1. No file i/o or network requests are made in the mutator.
+// 2. The validation is blocking for bundle deployments.
+//
+// The full suite of validation mutators is available in the [Validate] mutator.
+type fastValidateReadonly struct{}
+
+func FastValidateReadonly() bundle.ReadOnlyMutator {
+	return &fastValidateReadonly{}
+}
+
+func (f *fastValidateReadonly) Name() string {
+	return "fast_validate(readonly)"
+}
+
+func (f *fastValidateReadonly) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics {
+	return bundle.ApplyReadOnly(ctx, rb, bundle.Parallel(
+		// Fast mutators with only in-memory checks
+		JobClusterKeyDefined(),
+		JobTaskClusterSpec(),
+		SingleNodeCluster(),
+
+		// Blocking mutators. Deployments will fail if these checks fail.
+		ValidateArtifactPath(),
+	))
+}
+
+type fastValidate struct{}
+
+func FastValidate() bundle.Mutator {
+	return &fastValidate{}
+}
+
+func (f *fastValidate) Name() string {
+	return "fast_validate"
+}
+
+func (f *fastValidate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	return bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), FastValidateReadonly())
+}
diff --git a/bundle/config/validate/validate.go b/bundle/config/validate/validate.go
index 131566fc9..8fdd704ab 100644
--- a/bundle/config/validate/validate.go
+++ b/bundle/config/validate/validate.go
@@ -30,12 +30,13 @@ func (l location) Path() dyn.Path {
 // Apply implements bundle.Mutator.
 func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	return bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), bundle.Parallel(
-		JobClusterKeyDefined(),
+		FastValidateReadonly(),
+
+		// Slow mutators that require network or file i/o. These are only
+		// run in the `bundle validate` command.
 		FilesToSync(),
-		ValidateSyncPatterns(),
-		JobTaskClusterSpec(),
 		ValidateFolderPermissions(),
-		SingleNodeCluster(),
+		ValidateSyncPatterns(),
 	))
 }
 
diff --git a/bundle/config/validate/validate_artifact_path.go b/bundle/config/validate/validate_artifact_path.go
new file mode 100644
index 000000000..5bab99ccf
--- /dev/null
+++ b/bundle/config/validate/validate_artifact_path.go
@@ -0,0 +1,129 @@
+package validate
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/libraries"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/dynvar"
+	"github.com/databricks/databricks-sdk-go/apierr"
+)
+
+type validateArtifactPath struct{}
+
+func ValidateArtifactPath() bundle.ReadOnlyMutator {
+	return &validateArtifactPath{}
+}
+
+func (v *validateArtifactPath) Name() string {
+	return "validate:artifact_paths"
+}
+
+func extractVolumeFromPath(artifactPath string) (string, string, string, error) {
+	if !libraries.IsVolumesPath(artifactPath) {
+		return "", "", "", fmt.Errorf("expected artifact_path to start with /Volumes/, got %s", artifactPath)
+	}
+
+	parts := strings.Split(artifactPath, "/")
+	volumeFormatErr := fmt.Errorf("expected UC volume path to be in the format /Volumes/<catalog>/<schema>/<volume>/..., got %s", artifactPath)
+
+	// Incorrect format.
+	if len(parts) < 5 {
+		return "", "", "", volumeFormatErr
+	}
+
+	catalogName := parts[2]
+	schemaName := parts[3]
+	volumeName := parts[4]
+
+	// Incorrect format.
+	if catalogName == "" || schemaName == "" || volumeName == "" {
+		return "", "", "", volumeFormatErr
+	}
+
+	return catalogName, schemaName, volumeName, nil
+}
+
+func findVolumeInBundle(r config.Root, catalogName, schemaName, volumeName string) (dyn.Path, []dyn.Location, bool) {
+	volumes := r.Resources.Volumes
+	for k, v := range volumes {
+		if v.CatalogName != catalogName || v.Name != volumeName {
+			continue
+		}
+		// UC schemas can be defined in the bundle itself, and thus might be interpolated
+		// at runtime via the ${resources.schemas.<name>} syntax. Thus we match the volume
+		// definition if the schema name is the same as the one in the bundle, or if the
+		// schema name is interpolated.
+		// We only have to check for ${resources.schemas...} references because any
+		// other valid reference (like ${var.foo}) would have been interpolated by this point.
+		p, ok := dynvar.PureReferenceToPath(v.SchemaName)
+		isSchemaDefinedInBundle := ok && p.HasPrefix(dyn.Path{dyn.Key("resources"), dyn.Key("schemas")})
+		if v.SchemaName != schemaName && !isSchemaDefinedInBundle {
+			continue
+		}
+		pathString := fmt.Sprintf("resources.volumes.%s", k)
+		return dyn.MustPathFromString(pathString), r.GetLocations(pathString), true
+	}
+	return nil, nil, false
+}
+
+func (v *validateArtifactPath) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics {
+	// We only validate UC Volumes paths right now.
+	if !libraries.IsVolumesPath(rb.Config().Workspace.ArtifactPath) {
+		return nil
+	}
+
+	wrapErrorMsg := func(s string) diag.Diagnostics {
+		return diag.Diagnostics{
+			{
+				Summary:   s,
+				Severity:  diag.Error,
+				Locations: rb.Config().GetLocations("workspace.artifact_path"),
+				Paths:     []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")},
+			},
+		}
+	}
+
+	catalogName, schemaName, volumeName, err := extractVolumeFromPath(rb.Config().Workspace.ArtifactPath)
+	if err != nil {
+		return wrapErrorMsg(err.Error())
+	}
+	volumeFullName := fmt.Sprintf("%s.%s.%s", catalogName, schemaName, volumeName)
+	w := rb.WorkspaceClient()
+	_, err = w.Volumes.ReadByName(ctx, volumeFullName)
+
+	if errors.Is(err, apierr.ErrPermissionDenied) {
+		return wrapErrorMsg(fmt.Sprintf("cannot access volume %s: %s", volumeFullName, err))
+	}
+	if errors.Is(err, apierr.ErrNotFound) {
+		path, locations, ok := findVolumeInBundle(rb.Config(), catalogName, schemaName, volumeName)
+		if !ok {
+			return wrapErrorMsg(fmt.Sprintf("volume %s does not exist", volumeFullName))
+		}
+
+		// If the volume is defined in the bundle, provide a more helpful error diagnostic,
+		// with more details and location information.
+		return diag.Diagnostics{{
+			Summary:  fmt.Sprintf("volume %s does not exist", volumeFullName),
+			Severity: diag.Error,
+			Detail: `You are using a volume in your artifact_path that is managed by
+this bundle but which has not been deployed yet. Please first deploy
+the volume using 'bundle deploy' and then switch over to using it in
+the artifact_path.`,
+			Locations: slices.Concat(rb.Config().GetLocations("workspace.artifact_path"), locations),
+			Paths:     append([]dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, path),
+		}}
+
+	}
+	if err != nil {
+		return wrapErrorMsg(fmt.Sprintf("cannot read volume %s: %s", volumeFullName, err))
+	}
+	return nil
+}
diff --git a/bundle/config/validate/validate_artifact_path_test.go b/bundle/config/validate/validate_artifact_path_test.go
new file mode 100644
index 000000000..8fb5c9618
--- /dev/null
+++ b/bundle/config/validate/validate_artifact_path_test.go
@@ -0,0 +1,244 @@
+package validate
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/internal/bundletest"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/databricks-sdk-go/apierr"
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/service/catalog"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestValidateArtifactPathWithVolumeInBundle(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Workspace: config.Workspace{
+				ArtifactPath: "/Volumes/catalogN/schemaN/volumeN/abc",
+			},
+			Resources: config.Resources{
+				Volumes: map[string]*resources.Volume{
+					"foo": {
+						CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{
+							CatalogName: "catalogN",
+							Name:        "volumeN",
+							SchemaName:  "schemaN",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "file", Line: 1, Column: 1}})
+	bundletest.SetLocation(b, "resources.volumes.foo", []dyn.Location{{File: "file", Line: 2, Column: 2}})
+
+	ctx := context.Background()
+	m := mocks.NewMockWorkspaceClient(t)
+	api := m.GetMockVolumesAPI()
+	api.EXPECT().ReadByName(mock.Anything, "catalogN.schemaN.volumeN").Return(nil, &apierr.APIError{
+		StatusCode: 404,
+	})
+	b.SetWorkpaceClient(m.WorkspaceClient)
+
+	diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), ValidateArtifactPath())
+	assert.Equal(t, diag.Diagnostics{{
+		Severity: diag.Error,
+		Summary:  "volume catalogN.schemaN.volumeN does not exist",
+		Locations: []dyn.Location{
+			{File: "file", Line: 1, Column: 1},
+			{File: "file", Line: 2, Column: 2},
+		},
+		Paths: []dyn.Path{
+			dyn.MustPathFromString("workspace.artifact_path"),
+			dyn.MustPathFromString("resources.volumes.foo"),
+		},
+		Detail: `You are using a volume in your artifact_path that is managed by
+this bundle but which has not been deployed yet. Please first deploy
+the volume using 'bundle deploy' and then switch over to using it in
+the artifact_path.`,
+	}}, diags)
+}
+
+func TestValidateArtifactPath(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Workspace: config.Workspace{
+				ArtifactPath: "/Volumes/catalogN/schemaN/volumeN/abc",
+			},
+		},
+	}
+
+	bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "file", Line: 1, Column: 1}})
+	assertDiags := func(t *testing.T, diags diag.Diagnostics, expected string) {
+		assert.Len(t, diags, 1)
+		assert.Equal(t, diag.Diagnostics{{
+			Severity:  diag.Error,
+			Summary:   expected,
+			Locations: []dyn.Location{{File: "file", Line: 1, Column: 1}},
+			Paths:     []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")},
+		}}, diags)
+	}
+
+	rb := bundle.ReadOnly(b)
+	ctx := context.Background()
+
+	tcases := []struct {
+		err             error
+		expectedSummary string
+	}{
+		{
+			err: &apierr.APIError{
+				StatusCode: 403,
+				Message:    "User does not have USE SCHEMA on Schema 'catalogN.schemaN'",
+			},
+			expectedSummary: "cannot access volume catalogN.schemaN.volumeN: User does not have USE SCHEMA on Schema 'catalogN.schemaN'",
+		},
+		{
+			err: &apierr.APIError{
+				StatusCode: 404,
+			},
+			expectedSummary: "volume catalogN.schemaN.volumeN does not exist",
+		},
+		{
+			err: &apierr.APIError{
+				StatusCode: 500,
+				Message:    "Internal Server Error",
+			},
+			expectedSummary: "cannot read volume catalogN.schemaN.volumeN: Internal Server Error",
+		},
+	}
+
+	for _, tc := range tcases {
+		m := mocks.NewMockWorkspaceClient(t)
+		api := m.GetMockVolumesAPI()
+		api.EXPECT().ReadByName(mock.Anything, "catalogN.schemaN.volumeN").Return(nil, tc.err)
+		b.SetWorkpaceClient(m.WorkspaceClient)
+
+		diags := bundle.ApplyReadOnly(ctx, rb, ValidateArtifactPath())
+		assertDiags(t, diags, tc.expectedSummary)
+	}
+}
+
+func invalidVolumePaths() []string {
+	return []string{
+		"/Volumes/",
+		"/Volumes/main",
+		"/Volumes/main/",
+		"/Volumes/main//",
+		"/Volumes/main//my_schema",
+		"/Volumes/main/my_schema",
+		"/Volumes/main/my_schema/",
+		"/Volumes/main/my_schema//",
+		"/Volumes//my_schema/my_volume",
+	}
+}
+
+func TestExtractVolumeFromPath(t *testing.T) {
+	catalogName, schemaName, volumeName, err := extractVolumeFromPath("/Volumes/main/my_schema/my_volume")
+	require.NoError(t, err)
+	assert.Equal(t, "main", catalogName)
+	assert.Equal(t, "my_schema", schemaName)
+	assert.Equal(t, "my_volume", volumeName)
+
+	for _, p := range invalidVolumePaths() {
+		_, _, _, err := extractVolumeFromPath(p)
+		assert.EqualError(t, err, fmt.Sprintf("expected UC volume path to be in the format /Volumes/<catalog>/<schema>/<volume>/..., got %s", p))
+	}
+}
+
+func TestValidateArtifactPathWithInvalidPaths(t *testing.T) {
+	for _, p := range invalidVolumePaths() {
+		b := &bundle.Bundle{
+			Config: config.Root{
+				Workspace: config.Workspace{
+					ArtifactPath: p,
+				},
+			},
+		}
+
+		bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}})
+
+		diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), ValidateArtifactPath())
+		require.Equal(t, diag.Diagnostics{{
+			Severity:  diag.Error,
+			Summary:   fmt.Sprintf("expected UC volume path to be in the format /Volumes/<catalog>/<schema>/<volume>/..., got %s", p),
+			Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}},
+			Paths:     []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")},
+		}}, diags)
+	}
+}
+
+func TestFindVolumeInBundle(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Volumes: map[string]*resources.Volume{
+					"foo": {
+						CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{
+							CatalogName: "main",
+							Name:        "my_volume",
+							SchemaName:  "my_schema",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	bundletest.SetLocation(b, "resources.volumes.foo", []dyn.Location{
+		{
+			File:   "volume.yml",
+			Line:   1,
+			Column: 2,
+		},
+	})
+
+	// volume is in DAB.
+	path, locations, ok := findVolumeInBundle(b.Config, "main", "my_schema", "my_volume")
+	assert.True(t, ok)
+	assert.Equal(t, []dyn.Location{{
+		File:   "volume.yml",
+		Line:   1,
+		Column: 2,
+	}}, locations)
+	assert.Equal(t, dyn.MustPathFromString("resources.volumes.foo"), path)
+
+	// wrong volume name
+	_, _, ok = findVolumeInBundle(b.Config, "main", "my_schema", "doesnotexist")
+	assert.False(t, ok)
+
+	// wrong schema name
+	_, _, ok = findVolumeInBundle(b.Config, "main", "doesnotexist", "my_volume")
+	assert.False(t, ok)
+
+	// wrong catalog name
+	_, _, ok = findVolumeInBundle(b.Config, "doesnotexist", "my_schema", "my_volume")
+	assert.False(t, ok)
+
+	// schema name is interpolated but does not have the right prefix. In this case
+	// we should not match the volume.
+	b.Config.Resources.Volumes["foo"].SchemaName = "${foo.bar.baz}"
+	_, _, ok = findVolumeInBundle(b.Config, "main", "my_schema", "my_volume")
+	assert.False(t, ok)
+
+	// schema name is interpolated.
+	b.Config.Resources.Volumes["foo"].SchemaName = "${resources.schemas.my_schema.name}"
+	path, locations, ok = findVolumeInBundle(b.Config, "main", "valuedoesnotmatter", "my_volume")
+	assert.True(t, ok)
+	assert.Equal(t, []dyn.Location{{
+		File:   "volume.yml",
+		Line:   1,
+		Column: 2,
+	}}, locations)
+	assert.Equal(t, dyn.MustPathFromString("resources.volumes.foo"), path)
+}
diff --git a/bundle/libraries/filer.go b/bundle/libraries/filer.go
index 4448ed325..e09c75e0e 100644
--- a/bundle/libraries/filer.go
+++ b/bundle/libraries/filer.go
@@ -24,7 +24,7 @@ func GetFilerForLibraries(ctx context.Context, b *bundle.Bundle) (filer.Filer, s
 
 	switch {
 	case IsVolumesPath(artifactPath):
-		return filerForVolume(ctx, b)
+		return filerForVolume(b)
 
 	default:
 		return filerForWorkspace(b)
diff --git a/bundle/libraries/filer_test.go b/bundle/libraries/filer_test.go
index 88ba152fc..c18da9726 100644
--- a/bundle/libraries/filer_test.go
+++ b/bundle/libraries/filer_test.go
@@ -7,10 +7,7 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/libs/filer"
-	sdkconfig "github.com/databricks/databricks-sdk-go/config"
-	"github.com/databricks/databricks-sdk-go/experimental/mocks"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 )
 
@@ -39,11 +36,6 @@ func TestGetFilerForLibrariesValidUcVolume(t *testing.T) {
 		},
 	}
 
-	m := mocks.NewMockWorkspaceClient(t)
-	m.WorkspaceClient.Config = &sdkconfig.Config{}
-	m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/my_volume").Return(nil)
-	b.SetWorkpaceClient(m.WorkspaceClient)
-
 	client, uploadPath, diags := GetFilerForLibraries(context.Background(), b)
 	require.NoError(t, diags.Error())
 	assert.Equal(t, "/Volumes/main/my_schema/my_volume/.internal", uploadPath)
diff --git a/bundle/libraries/filer_volume.go b/bundle/libraries/filer_volume.go
index aecf68db1..176f475c6 100644
--- a/bundle/libraries/filer_volume.go
+++ b/bundle/libraries/filer_volume.go
@@ -1,132 +1,16 @@
 package libraries
 
 import (
-	"context"
-	"errors"
-	"fmt"
 	"path"
-	"strings"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/dyn"
-	"github.com/databricks/cli/libs/dyn/dynvar"
 	"github.com/databricks/cli/libs/filer"
-	"github.com/databricks/databricks-sdk-go/apierr"
 )
 
-func extractVolumeFromPath(artifactPath string) (string, string, string, error) {
-	if !IsVolumesPath(artifactPath) {
-		return "", "", "", fmt.Errorf("expected artifact_path to start with /Volumes/, got %s", artifactPath)
-	}
-
-	parts := strings.Split(artifactPath, "/")
-	volumeFormatErr := fmt.Errorf("expected UC volume path to be in the format /Volumes/<catalog>/<schema>/<volume>/..., got %s", artifactPath)
-
-	// Incorrect format.
-	if len(parts) < 5 {
-		return "", "", "", volumeFormatErr
-	}
-
-	catalogName := parts[2]
-	schemaName := parts[3]
-	volumeName := parts[4]
-
-	// Incorrect format.
-	if catalogName == "" || schemaName == "" || volumeName == "" {
-		return "", "", "", volumeFormatErr
-	}
-
-	return catalogName, schemaName, volumeName, nil
-}
-
-// This function returns a filer for ".internal" folder inside the directory configured
-// at `workspace.artifact_path`.
-// This function also checks if the UC volume exists in the workspace and then:
-//  1. If the UC volume exists in the workspace:
-//     Returns a filer for the UC volume.
-//  2. If the UC volume does not exist in the workspace but is (with high confidence) defined in
-//     the bundle configuration:
-//     Returns an error and a warning that instructs the user to deploy the
-//     UC volume before using it in the artifact path.
-//  3. If the UC volume does not exist in the workspace and is not defined in the bundle configuration:
-//     Returns an error.
-func filerForVolume(ctx context.Context, b *bundle.Bundle) (filer.Filer, string, diag.Diagnostics) {
-	artifactPath := b.Config.Workspace.ArtifactPath
+func filerForVolume(b *bundle.Bundle) (filer.Filer, string, diag.Diagnostics) {
 	w := b.WorkspaceClient()
-
-	catalogName, schemaName, volumeName, err := extractVolumeFromPath(artifactPath)
-	if err != nil {
-		return nil, "", diag.Diagnostics{
-			{
-				Severity:  diag.Error,
-				Summary:   err.Error(),
-				Locations: b.Config.GetLocations("workspace.artifact_path"),
-				Paths:     []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")},
-			},
-		}
-	}
-
-	// Check if the UC volume exists in the workspace.
-	volumePath := fmt.Sprintf("/Volumes/%s/%s/%s", catalogName, schemaName, volumeName)
-	err = w.Files.GetDirectoryMetadataByDirectoryPath(ctx, volumePath)
-
-	// If the volume exists already, directly return the filer for the path to
-	// upload the artifacts to.
-	if err == nil {
-		uploadPath := path.Join(artifactPath, InternalDirName)
-		f, err := filer.NewFilesClient(w, uploadPath)
-		return f, uploadPath, diag.FromErr(err)
-	}
-
-	baseErr := diag.Diagnostic{
-		Severity:  diag.Error,
-		Summary:   fmt.Sprintf("unable to determine if volume at %s exists: %s", volumePath, err),
-		Locations: b.Config.GetLocations("workspace.artifact_path"),
-		Paths:     []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")},
-	}
-
-	if errors.Is(err, apierr.ErrNotFound) {
-		// Since the API returned a 404, the volume does not exist.
-		// Modify the error message to provide more context.
-		baseErr.Summary = fmt.Sprintf("volume %s does not exist: %s", volumePath, err)
-
-		// If the volume is defined in the bundle, provide a more helpful error diagnostic,
-		// with more details and location information.
-		path, locations, ok := findVolumeInBundle(b, catalogName, schemaName, volumeName)
-		if !ok {
-			return nil, "", diag.Diagnostics{baseErr}
-		}
-		baseErr.Detail = `You are using a volume in your artifact_path that is managed by
-this bundle but which has not been deployed yet. Please first deploy
-the volume using 'bundle deploy' and then switch over to using it in
-the artifact_path.`
-		baseErr.Paths = append(baseErr.Paths, path)
-		baseErr.Locations = append(baseErr.Locations, locations...)
-	}
-
-	return nil, "", diag.Diagnostics{baseErr}
-}
-
-func findVolumeInBundle(b *bundle.Bundle, catalogName, schemaName, volumeName string) (dyn.Path, []dyn.Location, bool) {
-	volumes := b.Config.Resources.Volumes
-	for k, v := range volumes {
-		if v.CatalogName != catalogName || v.Name != volumeName {
-			continue
-		}
-		// UC schemas can be defined in the bundle itself, and thus might be interpolated
-		// at runtime via the ${resources.schemas.<name>} syntax. Thus we match the volume
-		// definition if the schema name is the same as the one in the bundle, or if the
-		// schema name is interpolated.
-		// We only have to check for ${resources.schemas...} references because any
-		// other valid reference (like ${var.foo}) would have been interpolated by this point.
-		p, ok := dynvar.PureReferenceToPath(v.SchemaName)
-		isSchemaDefinedInBundle := ok && p.HasPrefix(dyn.Path{dyn.Key("resources"), dyn.Key("schemas")})
-		if v.SchemaName != schemaName && !isSchemaDefinedInBundle {
-			continue
-		}
-		pathString := fmt.Sprintf("resources.volumes.%s", k)
-		return dyn.MustPathFromString(pathString), b.Config.GetLocations(pathString), true
-	}
-	return nil, nil, false
+	uploadPath := path.Join(b.Config.Workspace.ArtifactPath, InternalDirName)
+	f, err := filer.NewFilesClient(w, uploadPath)
+	return f, uploadPath, diag.FromErr(err)
 }
diff --git a/bundle/libraries/filer_volume_test.go b/bundle/libraries/filer_volume_test.go
index 2af54b0cb..39bdc4135 100644
--- a/bundle/libraries/filer_volume_test.go
+++ b/bundle/libraries/filer_volume_test.go
@@ -1,277 +1,27 @@
 package libraries
 
 import (
-	"context"
-	"fmt"
 	"path"
 	"testing"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
-	"github.com/databricks/cli/bundle/config/resources"
-	"github.com/databricks/cli/bundle/internal/bundletest"
-	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/filer"
-	"github.com/databricks/databricks-sdk-go/apierr"
-	sdkconfig "github.com/databricks/databricks-sdk-go/config"
-	"github.com/databricks/databricks-sdk-go/experimental/mocks"
-	"github.com/databricks/databricks-sdk-go/service/catalog"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 )
 
-func TestFindVolumeInBundle(t *testing.T) {
-	b := &bundle.Bundle{
-		Config: config.Root{
-			Resources: config.Resources{
-				Volumes: map[string]*resources.Volume{
-					"foo": {
-						CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{
-							CatalogName: "main",
-							Name:        "my_volume",
-							SchemaName:  "my_schema",
-						},
-					},
-				},
-			},
-		},
-	}
-
-	bundletest.SetLocation(b, "resources.volumes.foo", []dyn.Location{
-		{
-			File:   "volume.yml",
-			Line:   1,
-			Column: 2,
-		},
-	})
-
-	// volume is in DAB.
-	path, locations, ok := findVolumeInBundle(b, "main", "my_schema", "my_volume")
-	assert.True(t, ok)
-	assert.Equal(t, []dyn.Location{{
-		File:   "volume.yml",
-		Line:   1,
-		Column: 2,
-	}}, locations)
-	assert.Equal(t, dyn.MustPathFromString("resources.volumes.foo"), path)
-
-	// wrong volume name
-	_, _, ok = findVolumeInBundle(b, "main", "my_schema", "doesnotexist")
-	assert.False(t, ok)
-
-	// wrong schema name
-	_, _, ok = findVolumeInBundle(b, "main", "doesnotexist", "my_volume")
-	assert.False(t, ok)
-
-	// wrong catalog name
-	_, _, ok = findVolumeInBundle(b, "doesnotexist", "my_schema", "my_volume")
-	assert.False(t, ok)
-
-	// schema name is interpolated but does not have the right prefix. In this case
-	// we should not match the volume.
-	b.Config.Resources.Volumes["foo"].SchemaName = "${foo.bar.baz}"
-	_, _, ok = findVolumeInBundle(b, "main", "my_schema", "my_volume")
-	assert.False(t, ok)
-
-	// schema name is interpolated.
-	b.Config.Resources.Volumes["foo"].SchemaName = "${resources.schemas.my_schema.name}"
-	path, locations, ok = findVolumeInBundle(b, "main", "valuedoesnotmatter", "my_volume")
-	assert.True(t, ok)
-	assert.Equal(t, []dyn.Location{{
-		File:   "volume.yml",
-		Line:   1,
-		Column: 2,
-	}}, locations)
-	assert.Equal(t, dyn.MustPathFromString("resources.volumes.foo"), path)
-}
-
-func TestFilerForVolumeForErrorFromAPI(t *testing.T) {
+func TestFilerForVolume(t *testing.T) {
 	b := &bundle.Bundle{
 		Config: config.Root{
 			Workspace: config.Workspace{
-				ArtifactPath: "/Volumes/main/my_schema/my_volume",
+				ArtifactPath: "/Volumes/main/my_schema/my_volume/abc",
 			},
 		},
 	}
 
-	bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}})
-
-	m := mocks.NewMockWorkspaceClient(t)
-	m.WorkspaceClient.Config = &sdkconfig.Config{}
-	m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/my_volume").Return(fmt.Errorf("error from API"))
-	b.SetWorkpaceClient(m.WorkspaceClient)
-
-	_, _, diags := filerForVolume(context.Background(), b)
-	assert.Equal(t, diag.Diagnostics{
-		{
-			Severity:  diag.Error,
-			Summary:   "unable to determine if volume at /Volumes/main/my_schema/my_volume exists: error from API",
-			Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}},
-			Paths:     []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")},
-		},
-	}, diags)
-}
-
-func TestFilerForVolumeWithVolumeNotFound(t *testing.T) {
-	b := &bundle.Bundle{
-		Config: config.Root{
-			Workspace: config.Workspace{
-				ArtifactPath: "/Volumes/main/my_schema/doesnotexist",
-			},
-		},
-	}
-
-	bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}})
-
-	m := mocks.NewMockWorkspaceClient(t)
-	m.WorkspaceClient.Config = &sdkconfig.Config{}
-	m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/doesnotexist").Return(apierr.NotFound("some error message"))
-	b.SetWorkpaceClient(m.WorkspaceClient)
-
-	_, _, diags := filerForVolume(context.Background(), b)
-	assert.Equal(t, diag.Diagnostics{
-		{
-			Severity:  diag.Error,
-			Summary:   "volume /Volumes/main/my_schema/doesnotexist does not exist: some error message",
-			Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}},
-			Paths:     []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")},
-		},
-	}, diags)
-}
-
-func TestFilerForVolumeNotFoundAndInBundle(t *testing.T) {
-	b := &bundle.Bundle{
-		Config: config.Root{
-			Workspace: config.Workspace{
-				ArtifactPath: "/Volumes/main/my_schema/my_volume",
-			},
-			Resources: config.Resources{
-				Volumes: map[string]*resources.Volume{
-					"foo": {
-						CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{
-							CatalogName: "main",
-							Name:        "my_volume",
-							VolumeType:  "MANAGED",
-							SchemaName:  "my_schema",
-						},
-					},
-				},
-			},
-		},
-	}
-
-	bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}})
-	bundletest.SetLocation(b, "resources.volumes.foo", []dyn.Location{{File: "volume.yml", Line: 1, Column: 2}})
-
-	m := mocks.NewMockWorkspaceClient(t)
-	m.WorkspaceClient.Config = &sdkconfig.Config{}
-	m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/my_volume").Return(apierr.NotFound("error from API"))
-	b.SetWorkpaceClient(m.WorkspaceClient)
-
-	_, _, diags := GetFilerForLibraries(context.Background(), b)
-	assert.Equal(t, diag.Diagnostics{
-		{
-			Severity:  diag.Error,
-			Summary:   "volume /Volumes/main/my_schema/my_volume does not exist: error from API",
-			Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}, {File: "volume.yml", Line: 1, Column: 2}},
-			Paths:     []dyn.Path{dyn.MustPathFromString("workspace.artifact_path"), dyn.MustPathFromString("resources.volumes.foo")},
-			Detail: `You are using a volume in your artifact_path that is managed by
-this bundle but which has not been deployed yet. Please first deploy
-the volume using 'bundle deploy' and then switch over to using it in
-the artifact_path.`,
-		},
-	}, diags)
-}
-
-func invalidVolumePaths() []string {
-	return []string{
-		"/Volumes/",
-		"/Volumes/main",
-		"/Volumes/main/",
-		"/Volumes/main//",
-		"/Volumes/main//my_schema",
-		"/Volumes/main/my_schema",
-		"/Volumes/main/my_schema/",
-		"/Volumes/main/my_schema//",
-		"/Volumes//my_schema/my_volume",
-	}
-}
-
-func TestFilerForVolumeWithInvalidVolumePaths(t *testing.T) {
-	for _, p := range invalidVolumePaths() {
-		b := &bundle.Bundle{
-			Config: config.Root{
-				Workspace: config.Workspace{
-					ArtifactPath: p,
-				},
-			},
-		}
-
-		bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}})
-
-		_, _, diags := GetFilerForLibraries(context.Background(), b)
-		require.Equal(t, diag.Diagnostics{{
-			Severity:  diag.Error,
-			Summary:   fmt.Sprintf("expected UC volume path to be in the format /Volumes/<catalog>/<schema>/<volume>/..., got %s", p),
-			Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}},
-			Paths:     []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")},
-		}}, diags)
-	}
-}
-
-func TestFilerForVolumeWithInvalidPrefix(t *testing.T) {
-	b := &bundle.Bundle{
-		Config: config.Root{
-			Workspace: config.Workspace{
-				ArtifactPath: "/Volume/main/my_schema/my_volume",
-			},
-		},
-	}
-
-	_, _, diags := filerForVolume(context.Background(), b)
-	require.EqualError(t, diags.Error(), "expected artifact_path to start with /Volumes/, got /Volume/main/my_schema/my_volume")
-}
-
-func TestFilerForVolumeWithValidVolumePaths(t *testing.T) {
-	validPaths := []string{
-		"/Volumes/main/my_schema/my_volume",
-		"/Volumes/main/my_schema/my_volume/",
-		"/Volumes/main/my_schema/my_volume/a/b/c",
-		"/Volumes/main/my_schema/my_volume/a/a/a",
-	}
-
-	for _, p := range validPaths {
-		b := &bundle.Bundle{
-			Config: config.Root{
-				Workspace: config.Workspace{
-					ArtifactPath: p,
-				},
-			},
-		}
-
-		m := mocks.NewMockWorkspaceClient(t)
-		m.WorkspaceClient.Config = &sdkconfig.Config{}
-		m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/my_volume").Return(nil)
-		b.SetWorkpaceClient(m.WorkspaceClient)
-
-		client, uploadPath, diags := filerForVolume(context.Background(), b)
-		require.NoError(t, diags.Error())
-		assert.Equal(t, path.Join(p, ".internal"), uploadPath)
-		assert.IsType(t, &filer.FilesClient{}, client)
-	}
-}
-
-func TestExtractVolumeFromPath(t *testing.T) {
-	catalogName, schemaName, volumeName, err := extractVolumeFromPath("/Volumes/main/my_schema/my_volume")
-	require.NoError(t, err)
-	assert.Equal(t, "main", catalogName)
-	assert.Equal(t, "my_schema", schemaName)
-	assert.Equal(t, "my_volume", volumeName)
-
-	for _, p := range invalidVolumePaths() {
-		_, _, _, err := extractVolumeFromPath(p)
-		assert.EqualError(t, err, fmt.Sprintf("expected UC volume path to be in the format /Volumes/<catalog>/<schema>/<volume>/..., got %s", p))
-	}
+	client, uploadPath, diags := filerForVolume(b)
+	require.NoError(t, diags.Error())
+	assert.Equal(t, path.Join("/Volumes/main/my_schema/my_volume/abc/.internal"), uploadPath)
+	assert.IsType(t, &filer.FilesClient{}, client)
 }
diff --git a/bundle/libraries/upload_test.go b/bundle/libraries/upload_test.go
index 493785bf5..44b194c56 100644
--- a/bundle/libraries/upload_test.go
+++ b/bundle/libraries/upload_test.go
@@ -11,8 +11,6 @@ import (
 	mockfiler "github.com/databricks/cli/internal/mocks/libs/filer"
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/filer"
-	sdkconfig "github.com/databricks/databricks-sdk-go/config"
-	"github.com/databricks/databricks-sdk-go/experimental/mocks"
 	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/stretchr/testify/mock"
@@ -183,11 +181,6 @@ func TestArtifactUploadForVolumes(t *testing.T) {
 		filer.CreateParentDirectories,
 	).Return(nil)
 
-	m := mocks.NewMockWorkspaceClient(t)
-	m.WorkspaceClient.Config = &sdkconfig.Config{}
-	m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/foo/bar/artifacts").Return(nil)
-	b.SetWorkpaceClient(m.WorkspaceClient)
-
 	diags := bundle.Apply(context.Background(), b, bundle.Seq(ExpandGlobReferences(), UploadWithClient(mockFiler)))
 	require.NoError(t, diags.Error())
 
diff --git a/cmd/bundle/deploy.go b/cmd/bundle/deploy.go
index a25e02f6c..560b07e39 100644
--- a/cmd/bundle/deploy.go
+++ b/cmd/bundle/deploy.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config/validate"
 	"github.com/databricks/cli/bundle/phases"
 	"github.com/databricks/cli/bundle/render"
 	"github.com/databricks/cli/cmd/bundle/utils"
@@ -71,6 +72,7 @@ func newDeployCommand() *cobra.Command {
 			diags = diags.Extend(
 				bundle.Apply(ctx, b, bundle.Seq(
 					phases.Initialize(),
+					validate.FastValidate(),
 					phases.Build(),
 					phases.Deploy(outputHandler),
 				)),
diff --git a/integration/bundle/artifacts_test.go b/integration/bundle/artifacts_test.go
index 9dafe2f83..94b96899e 100644
--- a/integration/bundle/artifacts_test.go
+++ b/integration/bundle/artifacts_test.go
@@ -257,7 +257,7 @@ func TestUploadArtifactFileToVolumeThatDoesNotExist(t *testing.T) {
 	stdout, stderr, err := testcli.RequireErrorRun(t, ctx, "bundle", "deploy")
 
 	assert.Error(t, err)
-	assert.Equal(t, fmt.Sprintf(`Error: volume /Volumes/main/%s/doesnotexist does not exist: Not Found
+	assert.Equal(t, fmt.Sprintf(`Error: volume main.%s.doesnotexist does not exist
   at workspace.artifact_path
   in databricks.yml:6:18
 
@@ -293,7 +293,7 @@ func TestUploadArtifactToVolumeNotYetDeployed(t *testing.T) {
 	stdout, stderr, err := testcli.RequireErrorRun(t, ctx, "bundle", "deploy")
 
 	assert.Error(t, err)
-	assert.Equal(t, fmt.Sprintf(`Error: volume /Volumes/main/%s/my_volume does not exist: Not Found
+	assert.Equal(t, fmt.Sprintf(`Error: volume main.%s.my_volume does not exist
   at workspace.artifact_path
      resources.volumes.foo
   in databricks.yml:6:18

From 60782b57bdbe09af7cfe1232b827b4668ca92bb9 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Thu, 2 Jan 2025 14:23:00 +0100
Subject: [PATCH 019/247] Added close stale issues workflow (#634)

## Changes
Added workflows for closing stale issues.

It adds a Github Action that warns and then auto closes stale issues.
---
 .github/workflows/close-stale-issues.yml | 36 ++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 .github/workflows/close-stale-issues.yml

diff --git a/.github/workflows/close-stale-issues.yml b/.github/workflows/close-stale-issues.yml
new file mode 100644
index 000000000..ffe550132
--- /dev/null
+++ b/.github/workflows/close-stale-issues.yml
@@ -0,0 +1,36 @@
+name: "Close Stale Issues"
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * *" # Run at midnight every day
+
+jobs:
+  cleanup:
+    permissions:
+      issues: write
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-latest
+    name: Stale issue job
+    steps:
+      - uses: actions/stale@v9
+        with:
+          stale-issue-message: This issue has not received a response in a while. If you want to keep this issue open, please leave a comment below and auto-close will be canceled.
+          stale-pr-message: This PR has not received an update in a while. If you want to keep this PR open, please leave a comment below or push a new commit and auto-close will be canceled.
+
+          # These labels are required
+          stale-issue-label: Stale
+          stale-pr-label: Stale
+
+          exempt-issue-labels: No Autoclose
+          exempt-pr-labels: No Autoclose
+
+          # Issue timing
+          days-before-stale: 30
+          days-before-close: 7
+
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          loglevel: DEBUG
+          # TODO: Remove dry-run after merge when confirmed it works correctly
+          dry-run: true

From 39d1e8093fcaf359c617c4f5580123a43f7f7ec9 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 3 Jan 2025 10:25:07 +0100
Subject: [PATCH 020/247] Enable intrange linter and apply autofix (#2069)

New construct in Go1.22+ for integer iteration:
https://github.com/ckaznocha/intrange?tab=readme-ov-file#intrange
---
 .golangci.yaml                                    |  1 +
 bundle/config/mutator/process_target_mode_test.go |  4 ++--
 bundle/config/resources_test.go                   |  6 +++---
 bundle/config/variable/lookup_test.go             |  2 +-
 bundle/deploy/terraform/convert_test.go           |  2 +-
 bundle/deploy/terraform/state_push_test.go        |  2 +-
 bundle/internal/schema/parser.go                  |  2 +-
 bundle/run/pipeline.go                            |  4 ++--
 integration/libs/locker/locker_test.go            | 12 ++++++------
 integration/python/python_tasks_test.go           |  6 +++---
 libs/cmdio/render.go                              |  2 +-
 libs/cmdio/render_test.go                         |  2 +-
 libs/dyn/convert/from_typed.go                    |  2 +-
 libs/dyn/convert/struct_info.go                   |  2 +-
 libs/dyn/mapping_test.go                          |  2 +-
 libs/dyn/merge/override.go                        |  2 +-
 libs/dyn/path.go                                  |  4 ++--
 libs/dyn/visit_map_test.go                        |  8 ++++----
 libs/exec/exec_test.go                            |  2 +-
 libs/filer/fs_test.go                             |  2 +-
 libs/jsonschema/from_type.go                      |  4 ++--
 21 files changed, 37 insertions(+), 36 deletions(-)

diff --git a/.golangci.yaml b/.golangci.yaml
index 3e9a88957..6558c502d 100644
--- a/.golangci.yaml
+++ b/.golangci.yaml
@@ -12,6 +12,7 @@ linters:
     - gofumpt
     - goimports
     - testifylint
+    - intrange
 linters-settings:
   govet:
     enable-all: true
diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go
index e1aa9e59b..c299a7636 100644
--- a/bundle/config/mutator/process_target_mode_test.go
+++ b/bundle/config/mutator/process_target_mode_test.go
@@ -382,7 +382,7 @@ func TestAllResourcesMocked(t *testing.T) {
 	b := mockBundle(config.Development)
 	resources := reflect.ValueOf(b.Config.Resources)
 
-	for i := 0; i < resources.NumField(); i++ {
+	for i := range resources.NumField() {
 		field := resources.Field(i)
 		if field.Kind() == reflect.Map {
 			assert.True(
@@ -411,7 +411,7 @@ func TestAllNonUcResourcesAreRenamed(t *testing.T) {
 	require.NoError(t, diags.Error())
 
 	resources := reflect.ValueOf(b.Config.Resources)
-	for i := 0; i < resources.NumField(); i++ {
+	for i := range resources.NumField() {
 		field := resources.Field(i)
 
 		if field.Kind() == reflect.Map {
diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go
index 3da645585..cbbcf5e27 100644
--- a/bundle/config/resources_test.go
+++ b/bundle/config/resources_test.go
@@ -33,7 +33,7 @@ func TestCustomMarshallerIsImplemented(t *testing.T) {
 	r := Resources{}
 	rt := reflect.TypeOf(r)
 
-	for i := 0; i < rt.NumField(); i++ {
+	for i := range rt.NumField() {
 		field := rt.Field(i)
 
 		// Fields in Resources are expected be of the form map[string]*resourceStruct
@@ -75,7 +75,7 @@ func TestResourcesAllResourcesCompleteness(t *testing.T) {
 		types = append(types, group.Description.PluralName)
 	}
 
-	for i := 0; i < rt.NumField(); i++ {
+	for i := range rt.NumField() {
 		field := rt.Field(i)
 		jsonTag := field.Tag.Get("json")
 
@@ -92,7 +92,7 @@ func TestSupportedResources(t *testing.T) {
 	actual := SupportedResources()
 
 	typ := reflect.TypeOf(Resources{})
-	for i := 0; i < typ.NumField(); i++ {
+	for i := range typ.NumField() {
 		field := typ.Field(i)
 		jsonTags := strings.Split(field.Tag.Get("json"), ",")
 		pluralName := jsonTags[0]
diff --git a/bundle/config/variable/lookup_test.go b/bundle/config/variable/lookup_test.go
index bd54d89fc..bcfcb4626 100644
--- a/bundle/config/variable/lookup_test.go
+++ b/bundle/config/variable/lookup_test.go
@@ -13,7 +13,7 @@ func TestLookup_Coverage(t *testing.T) {
 	val := reflect.ValueOf(lookup)
 	typ := val.Type()
 
-	for i := 0; i < val.NumField(); i++ {
+	for i := range val.NumField() {
 		field := val.Field(i)
 		if field.Kind() != reflect.String {
 			t.Fatalf("Field %s is not a string", typ.Field(i).Name)
diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go
index 84b3c5788..ccfdcece3 100644
--- a/bundle/deploy/terraform/convert_test.go
+++ b/bundle/deploy/terraform/convert_test.go
@@ -1261,7 +1261,7 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
 
 func AssertFullResourceCoverage(t *testing.T, config *config.Root) {
 	resources := reflect.ValueOf(config.Resources)
-	for i := 0; i < resources.NumField(); i++ {
+	for i := range resources.NumField() {
 		field := resources.Field(i)
 		if field.Kind() == reflect.Map {
 			assert.True(
diff --git a/bundle/deploy/terraform/state_push_test.go b/bundle/deploy/terraform/state_push_test.go
index 4cc52b7a7..54e7f621c 100644
--- a/bundle/deploy/terraform/state_push_test.go
+++ b/bundle/deploy/terraform/state_push_test.go
@@ -71,7 +71,7 @@ func TestStatePushLargeState(t *testing.T) {
 	b := statePushTestBundle(t)
 
 	largeState := map[string]any{}
-	for i := 0; i < 1000000; i++ {
+	for i := range 1000000 {
 		largeState[fmt.Sprintf("field_%d", i)] = i
 	}
 
diff --git a/bundle/internal/schema/parser.go b/bundle/internal/schema/parser.go
index e1d1a13dc..919908429 100644
--- a/bundle/internal/schema/parser.go
+++ b/bundle/internal/schema/parser.go
@@ -54,7 +54,7 @@ func (p *openapiParser) findRef(typ reflect.Type) (jsonschema.Schema, bool) {
 
 	// Check for embedded Databricks Go SDK types.
 	if typ.Kind() == reflect.Struct {
-		for i := 0; i < typ.NumField(); i++ {
+		for i := range typ.NumField() {
 			if !typ.Field(i).Anonymous {
 				continue
 			}
diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go
index d84015d76..c447f044a 100644
--- a/bundle/run/pipeline.go
+++ b/bundle/run/pipeline.go
@@ -17,7 +17,7 @@ import (
 
 func filterEventsByUpdateId(events []pipelines.PipelineEvent, updateId string) []pipelines.PipelineEvent {
 	result := []pipelines.PipelineEvent{}
-	for i := 0; i < len(events); i++ {
+	for i := range events {
 		if events[i].Origin.UpdateId == updateId {
 			result = append(result, events[i])
 		}
@@ -32,7 +32,7 @@ func (r *pipelineRunner) logEvent(ctx context.Context, event pipelines.PipelineE
 	}
 	if event.Error != nil && len(event.Error.Exceptions) > 0 {
 		logString += "trace for most recent exception: \n"
-		for i := 0; i < len(event.Error.Exceptions); i++ {
+		for i := range len(event.Error.Exceptions) {
 			logString += fmt.Sprintf("%s\n", event.Error.Exceptions[i].Message)
 		}
 	}
diff --git a/integration/libs/locker/locker_test.go b/integration/libs/locker/locker_test.go
index c51972b90..524996465 100644
--- a/integration/libs/locker/locker_test.go
+++ b/integration/libs/locker/locker_test.go
@@ -60,13 +60,13 @@ func TestLock(t *testing.T) {
 
 	lockerErrs := make([]error, numConcurrentLocks)
 	lockers := make([]*lockpkg.Locker, numConcurrentLocks)
-	for i := 0; i < numConcurrentLocks; i++ {
+	for i := range numConcurrentLocks {
 		lockers[i], err = lockpkg.CreateLocker("humpty.dumpty@databricks.com", remoteProjectRoot, wsc)
 		require.NoError(t, err)
 	}
 
 	var wg sync.WaitGroup
-	for i := 0; i < numConcurrentLocks; i++ {
+	for i := range numConcurrentLocks {
 		wg.Add(1)
 		currentIndex := i
 		go func() {
@@ -80,7 +80,7 @@ func TestLock(t *testing.T) {
 	countActive := 0
 	indexOfActiveLocker := 0
 	indexOfAnInactiveLocker := -1
-	for i := 0; i < numConcurrentLocks; i++ {
+	for i := range numConcurrentLocks {
 		if lockers[i].Active {
 			countActive += 1
 			assert.NoError(t, lockerErrs[i])
@@ -102,7 +102,7 @@ func TestLock(t *testing.T) {
 	assert.True(t, remoteLocker.AcquisitionTime.Equal(lockers[indexOfActiveLocker].State.AcquisitionTime), "remote locker acquisition time does not match active locker")
 
 	// test all other locks (inactive ones) do not match the remote lock and Unlock fails
-	for i := 0; i < numConcurrentLocks; i++ {
+	for i := range numConcurrentLocks {
 		if i == indexOfActiveLocker {
 			continue
 		}
@@ -112,7 +112,7 @@ func TestLock(t *testing.T) {
 	}
 
 	// test inactive locks fail to write a file
-	for i := 0; i < numConcurrentLocks; i++ {
+	for i := range numConcurrentLocks {
 		if i == indexOfActiveLocker {
 			continue
 		}
@@ -140,7 +140,7 @@ func TestLock(t *testing.T) {
 	assert.Equal(t, "Shah Rukh", res["name"])
 
 	// inactive locker file reads fail
-	for i := 0; i < numConcurrentLocks; i++ {
+	for i := range numConcurrentLocks {
 		if i == indexOfActiveLocker {
 			continue
 		}
diff --git a/integration/python/python_tasks_test.go b/integration/python/python_tasks_test.go
index 9ad3ed5de..9411afb13 100644
--- a/integration/python/python_tasks_test.go
+++ b/integration/python/python_tasks_test.go
@@ -266,7 +266,7 @@ func prepareRepoFiles(t *testing.T) *testFiles {
 
 func GenerateNotebookTasks(notebookPath string, versions []string, nodeTypeId string) []jobs.SubmitTask {
 	tasks := make([]jobs.SubmitTask, 0)
-	for i := 0; i < len(versions); i++ {
+	for i := range versions {
 		task := jobs.SubmitTask{
 			TaskKey: fmt.Sprintf("notebook_%s", strings.ReplaceAll(versions[i], ".", "_")),
 			NotebookTask: &jobs.NotebookTask{
@@ -287,7 +287,7 @@ func GenerateNotebookTasks(notebookPath string, versions []string, nodeTypeId st
 
 func GenerateSparkPythonTasks(notebookPath string, versions []string, nodeTypeId string) []jobs.SubmitTask {
 	tasks := make([]jobs.SubmitTask, 0)
-	for i := 0; i < len(versions); i++ {
+	for i := range versions {
 		task := jobs.SubmitTask{
 			TaskKey: fmt.Sprintf("spark_%s", strings.ReplaceAll(versions[i], ".", "_")),
 			SparkPythonTask: &jobs.SparkPythonTask{
@@ -308,7 +308,7 @@ func GenerateSparkPythonTasks(notebookPath string, versions []string, nodeTypeId
 
 func GenerateWheelTasks(wheelPath string, versions []string, nodeTypeId string) []jobs.SubmitTask {
 	tasks := make([]jobs.SubmitTask, 0)
-	for i := 0; i < len(versions); i++ {
+	for i := range versions {
 		task := jobs.SubmitTask{
 			TaskKey: fmt.Sprintf("whl_%s", strings.ReplaceAll(versions[i], ".", "_")),
 			PythonWheelTask: &jobs.PythonWheelTask{
diff --git a/libs/cmdio/render.go b/libs/cmdio/render.go
index 1529274a3..1a6aadcfa 100644
--- a/libs/cmdio/render.go
+++ b/libs/cmdio/render.go
@@ -39,7 +39,7 @@ func Heredoc(tmpl string) (trimmed string) {
 			break
 		}
 	}
-	for i := 0; i < len(lines); i++ {
+	for i := range lines {
 		if lines[i] == "" || strings.TrimSpace(lines[i]) == "" {
 			continue
 		}
diff --git a/libs/cmdio/render_test.go b/libs/cmdio/render_test.go
index f26190a23..1c9d1c93b 100644
--- a/libs/cmdio/render_test.go
+++ b/libs/cmdio/render_test.go
@@ -55,7 +55,7 @@ func (d *dummyIterator) Next(ctx context.Context) (*provisioning.Workspace, erro
 func makeWorkspaces(count int) []*provisioning.Workspace {
 	res := make([]*provisioning.Workspace, 0, count)
 	next := []*provisioning.Workspace{&dummyWorkspace1, &dummyWorkspace2}
-	for i := 0; i < count; i++ {
+	for range count {
 		n := next[0]
 		next = append(next[1:], n)
 		res = append(res, n)
diff --git a/libs/dyn/convert/from_typed.go b/libs/dyn/convert/from_typed.go
index ed1b85a36..3de017b75 100644
--- a/libs/dyn/convert/from_typed.go
+++ b/libs/dyn/convert/from_typed.go
@@ -209,7 +209,7 @@ func fromTypedSlice(src reflect.Value, ref dyn.Value) (dyn.Value, error) {
 	}
 
 	out := make([]dyn.Value, src.Len())
-	for i := 0; i < src.Len(); i++ {
+	for i := range src.Len() {
 		v := src.Index(i)
 		refv := ref.Index(i)
 
diff --git a/libs/dyn/convert/struct_info.go b/libs/dyn/convert/struct_info.go
index f5fd29cb9..1e34008e2 100644
--- a/libs/dyn/convert/struct_info.go
+++ b/libs/dyn/convert/struct_info.go
@@ -65,7 +65,7 @@ func buildStructInfo(typ reflect.Type) structInfo {
 		}
 
 		nf := styp.NumField()
-		for j := 0; j < nf; j++ {
+		for j := range nf {
 			sf := styp.Field(j)
 
 			// Recurse into anonymous fields.
diff --git a/libs/dyn/mapping_test.go b/libs/dyn/mapping_test.go
index 43b24b0c5..67144ae55 100644
--- a/libs/dyn/mapping_test.go
+++ b/libs/dyn/mapping_test.go
@@ -185,7 +185,7 @@ func TestMappingClone(t *testing.T) {
 
 func TestMappingMerge(t *testing.T) {
 	var m1 dyn.Mapping
-	for i := 0; i < 10; i++ {
+	for i := range 10 {
 		err := m1.Set(dyn.V(fmt.Sprintf("%d", i)), dyn.V(i))
 		require.NoError(t, err)
 	}
diff --git a/libs/dyn/merge/override.go b/libs/dyn/merge/override.go
index ca62c7305..1e49d5544 100644
--- a/libs/dyn/merge/override.go
+++ b/libs/dyn/merge/override.go
@@ -165,7 +165,7 @@ func overrideSequence(basePath dyn.Path, left, right []dyn.Value, visitor Overri
 	minLen := min(len(left), len(right))
 	var values []dyn.Value
 
-	for i := 0; i < minLen; i++ {
+	for i := range minLen {
 		path := basePath.Append(dyn.Index(i))
 		merged, err := override(path, left[i], right[i], visitor)
 		if err != nil {
diff --git a/libs/dyn/path.go b/libs/dyn/path.go
index 76377e2dc..25bff5070 100644
--- a/libs/dyn/path.go
+++ b/libs/dyn/path.go
@@ -65,7 +65,7 @@ func (p Path) Equal(q Path) bool {
 	if pl != ql {
 		return false
 	}
-	for i := 0; i < pl; i++ {
+	for i := range pl {
 		if p[i] != q[i] {
 			return false
 		}
@@ -81,7 +81,7 @@ func (p Path) HasPrefix(q Path) bool {
 	if pl < ql {
 		return false
 	}
-	for i := 0; i < ql; i++ {
+	for i := range ql {
 		if p[i] != q[i] {
 			return false
 		}
diff --git a/libs/dyn/visit_map_test.go b/libs/dyn/visit_map_test.go
index d62327d6f..3c2908c4b 100644
--- a/libs/dyn/visit_map_test.go
+++ b/libs/dyn/visit_map_test.go
@@ -87,12 +87,12 @@ func TestMapFuncOnMapWithEmptySequence(t *testing.T) {
 		dyn.V([]dyn.Value{dyn.V(42)}),
 	}
 
-	for i := 0; i < len(variants); i++ {
+	for i := range variants {
 		vin := dyn.V(map[string]dyn.Value{
 			"key": variants[i],
 		})
 
-		for j := 0; j < len(variants); j++ {
+		for j := range variants {
 			vout, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Key("key")), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
 				return variants[j], nil
 			})
@@ -153,12 +153,12 @@ func TestMapFuncOnSequenceWithEmptySequence(t *testing.T) {
 		dyn.V([]dyn.Value{dyn.V(42)}),
 	}
 
-	for i := 0; i < len(variants); i++ {
+	for i := range variants {
 		vin := dyn.V([]dyn.Value{
 			variants[i],
 		})
 
-		for j := 0; j < len(variants); j++ {
+		for j := range variants {
 			vout, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Index(0)), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
 				return variants[j], nil
 			})
diff --git a/libs/exec/exec_test.go b/libs/exec/exec_test.go
index e75c158bd..c363c1f7c 100644
--- a/libs/exec/exec_test.go
+++ b/libs/exec/exec_test.go
@@ -141,7 +141,7 @@ func TestMultipleCommandsRunInParrallel(t *testing.T) {
 	const count = 5
 	var wg sync.WaitGroup
 
-	for i := 0; i < count; i++ {
+	for i := range count {
 		wg.Add(1)
 		cmd, err := executor.StartCommand(context.Background(), fmt.Sprintf("echo 'Hello %d'", i))
 		go func(cmd Command, i int) {
diff --git a/libs/filer/fs_test.go b/libs/filer/fs_test.go
index 08d7a9428..6168af39a 100644
--- a/libs/filer/fs_test.go
+++ b/libs/filer/fs_test.go
@@ -107,7 +107,7 @@ func TestFsOpenDir(t *testing.T) {
 
 		de.Close()
 
-		for i := 0; i < 3; i++ {
+		for range 3 {
 			tmp, err = de.ReadDir(1)
 			require.NoError(t, err)
 			entries = append(entries, tmp...)
diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go
index 18a2b3ba5..6f8f39d96 100644
--- a/libs/jsonschema/from_type.go
+++ b/libs/jsonschema/from_type.go
@@ -211,7 +211,7 @@ func getStructFields(typ reflect.Type) []reflect.StructField {
 	fields := []reflect.StructField{}
 	bfsQueue := list.New()
 
-	for i := 0; i < typ.NumField(); i++ {
+	for i := range typ.NumField() {
 		bfsQueue.PushBack(typ.Field(i))
 	}
 	for bfsQueue.Len() > 0 {
@@ -233,7 +233,7 @@ func getStructFields(typ reflect.Type) []reflect.StructField {
 			fieldType = fieldType.Elem()
 		}
 
-		for i := 0; i < fieldType.NumField(); i++ {
+		for i := range fieldType.NumField() {
 			bfsQueue.PushBack(fieldType.Field(i))
 		}
 	}

From 8e8399da8312c127bcfc3760efafc60d1a501214 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 3 Jan 2025 11:13:12 +0100
Subject: [PATCH 021/247] Enable linter 'mirror' and autofix existing issues
 (#2070)

https://github.com/butuzov/mirror
---
 .golangci.yaml                       | 1 +
 bundle/deploy/terraform/import.go    | 3 ++-
 libs/cmdio/render_test.go            | 2 +-
 libs/filer/workspace_files_client.go | 2 +-
 4 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.golangci.yaml b/.golangci.yaml
index 6558c502d..06d8152e5 100644
--- a/.golangci.yaml
+++ b/.golangci.yaml
@@ -13,6 +13,7 @@ linters:
     - goimports
     - testifylint
     - intrange
+    - mirror
 linters-settings:
   govet:
     enable-all: true
diff --git a/bundle/deploy/terraform/import.go b/bundle/deploy/terraform/import.go
index 0a1d1b9ce..a0604e71d 100644
--- a/bundle/deploy/terraform/import.go
+++ b/bundle/deploy/terraform/import.go
@@ -7,6 +7,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/libs/cmdio"
@@ -67,7 +68,7 @@ func (m *importResource) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn
 	if changed && !m.opts.AutoApprove {
 		output := buf.String()
 		// Remove output starting from Warning until end of output
-		output = output[:bytes.Index([]byte(output), []byte("Warning:"))]
+		output = output[:strings.Index(output, "Warning:")]
 		cmdio.LogString(ctx, output)
 
 		if !cmdio.IsPromptSupported(ctx) {
diff --git a/libs/cmdio/render_test.go b/libs/cmdio/render_test.go
index 1c9d1c93b..51b385b1d 100644
--- a/libs/cmdio/render_test.go
+++ b/libs/cmdio/render_test.go
@@ -74,7 +74,7 @@ func makeIterator(count int) listing.Iterator[*provisioning.Workspace] {
 func makeBigOutput(count int) string {
 	res := bytes.Buffer{}
 	for _, ws := range makeWorkspaces(count) {
-		res.Write([]byte(fmt.Sprintf("%d  %s\n", ws.WorkspaceId, ws.WorkspaceName)))
+		res.WriteString(fmt.Sprintf("%d  %s\n", ws.WorkspaceId, ws.WorkspaceName))
 	}
 	return res.String()
 }
diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go
index 9e0a7ce50..8d5148edd 100644
--- a/libs/filer/workspace_files_client.go
+++ b/libs/filer/workspace_files_client.go
@@ -195,7 +195,7 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io
 
 	// This API returns 400 if the file already exists, when the object type is notebook
 	regex := regexp.MustCompile(`Path \((.*)\) already exists.`)
-	if aerr.StatusCode == http.StatusBadRequest && regex.Match([]byte(aerr.Message)) {
+	if aerr.StatusCode == http.StatusBadRequest && regex.MatchString(aerr.Message) {
 		// Parse file path from regex capture group
 		matches := regex.FindStringSubmatch(aerr.Message)
 		if len(matches) == 2 {

From 9abb11decbc9e2293d5e4b3c6a4eabb7ffeca21c Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Fri, 3 Jan 2025 16:05:20 +0530
Subject: [PATCH 022/247] Enable TestBundleInitOnMlopsStacks (#2072)

## Changes
This test was broken due to upstream change:
https://github.com/databricks/mlops-stacks/pull/187

Fixed in upstream change:
https://github.com/databricks/mlops-stacks/pull/188

## Tests
Test passes now
---
 integration/bundle/init_test.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/integration/bundle/init_test.go b/integration/bundle/init_test.go
index bc81cd3a8..f5c263ca3 100644
--- a/integration/bundle/init_test.go
+++ b/integration/bundle/init_test.go
@@ -39,8 +39,6 @@ func TestBundleInitErrorOnUnknownFields(t *testing.T) {
 //     make changes that can break the MLOps Stacks DAB. In which case we should
 //     skip this test until the MLOps Stacks DAB is updated to work again.
 func TestBundleInitOnMlopsStacks(t *testing.T) {
-	testutil.SkipUntil(t, "2025-01-09")
-
 	ctx, wt := acc.WorkspaceTest(t)
 	w := wt.W
 

From ab6b1f1d772fa84adc6fd8cab954fbfd5f38a1ba Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 3 Jan 2025 12:32:03 +0100
Subject: [PATCH 023/247] Upgrade golang.org/x/net from v0.26.0 to v0.33.0
 (#2073)

Fixes https://github.com/databricks/cli/security/dependabot/20
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 2dda0cd60..86bc1c368 100644
--- a/go.mod
+++ b/go.mod
@@ -68,7 +68,7 @@ require (
 	go.opentelemetry.io/otel/metric v1.24.0 // indirect
 	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	golang.org/x/crypto v0.31.0 // indirect
-	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/net v0.33.0 // indirect
 	golang.org/x/sys v0.28.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	google.golang.org/api v0.182.0 // indirect
diff --git a/go.sum b/go.sum
index 1e806ea03..f6cf79607 100644
--- a/go.sum
+++ b/go.sum
@@ -204,8 +204,8 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
-golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
 golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=

From 4f3cf6ac2cad002ac3e6d24dbda657b4c22fb314 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Fri, 3 Jan 2025 15:12:55 +0100
Subject: [PATCH 024/247] Limit comment about integration tests to PRs from
 forks (#2075)

## Changes

The comment block appears on all PRs, even if the integration tests are
automatically triggered. This is quite noisy. This change limits those
comments to PRs from forks.

## Tests

Have to try by merging...
---
 .github/workflows/external-message.yml |  6 +++++-
 .github/workflows/integration-pr.yml   | 25 +++----------------------
 2 files changed, 8 insertions(+), 23 deletions(-)

diff --git a/.github/workflows/external-message.yml b/.github/workflows/external-message.yml
index 1970735f9..28e61503c 100644
--- a/.github/workflows/external-message.yml
+++ b/.github/workflows/external-message.yml
@@ -17,6 +17,10 @@ jobs:
     permissions:
       pull-requests: write
 
+    # Only run this job for PRs from forks.
+    # Integration tests are not run automatically for PRs from forks.
+    if: "${{ github.event.pull_request.head.repo.fork }}"
+
     steps:
       - uses: actions/checkout@v4
 
@@ -43,7 +47,7 @@ jobs:
         run: |
           gh pr comment ${{ github.event.pull_request.number }} --body \
           "<!-- INTEGRATION_TESTS_MANUAL -->
-          If integration tests don't run automatically, an authorized user can run them manually by following the instructions below:
+          An authorized user can trigger integration tests manually by following the instructions below:
 
           Trigger:
           [go/deco-tests-run/cli](https://go/deco-tests-run/cli)
diff --git a/.github/workflows/integration-pr.yml b/.github/workflows/integration-pr.yml
index bf2dcd8bc..c1e3a9a29 100644
--- a/.github/workflows/integration-pr.yml
+++ b/.github/workflows/integration-pr.yml
@@ -5,36 +5,17 @@ on:
     types: [opened, synchronize]
 
 jobs:
-  check-token:
-    runs-on: ubuntu-latest
-    environment: "test-trigger-is"
-
-    outputs:
-      has_token: ${{ steps.set-token-status.outputs.has_token }}
-
-    steps:
-      - name: Check if DECO_WORKFLOW_TRIGGER_APP_ID is set
-        id: set-token-status
-        run: |
-          if [ -z "${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}" ]; then
-            echo "DECO_WORKFLOW_TRIGGER_APP_ID is empty. User has no access to secrets."
-            echo "::set-output name=has_token::false"
-          else
-            echo "DECO_WORKFLOW_TRIGGER_APP_ID is set. User has access to secrets."
-            echo "::set-output name=has_token::true"
-          fi
-
   # Trigger for pull requests.
   #
   # This workflow triggers the integration test workflow in a different repository.
   # It requires secrets from the "test-trigger-is" environment, which are only available to authorized users.
-  # It depends on the "check-token" workflow to confirm access to this environment to avoid failures.
   trigger:
     runs-on: ubuntu-latest
     environment: "test-trigger-is"
 
-    if: needs.check-token.outputs.has_token == 'true'
-    needs: check-token
+    # Only run this job for PRs from branches on the main repository and not from forks.
+    # Workflows triggered by PRs from forks don't have access to the "test-trigger-is" environment.
+    if: "${{ !github.event.pull_request.head.repo.fork }}"
 
     steps:
       - name: Generate GitHub App Token

From 8af98accee855eb336a8f658d6d63e6de278e132 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 6 Jan 2025 14:12:34 +0100
Subject: [PATCH 025/247] Set gotestsum --format to github-actions when running
 on github (#2082)

This gives grouping and ability to open individual test logs.
---
 .github/workflows/push.yml | 3 +++
 Makefile                   | 6 ++++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
index f7c4d5456..a4a35420a 100644
--- a/.github/workflows/push.yml
+++ b/.github/workflows/push.yml
@@ -14,6 +14,9 @@ on:
     branches:
       - main
 
+env:
+  GOTESTSUM_FORMAT: github-actions
+
 jobs:
   tests:
     runs-on: ${{ matrix.os }}
diff --git a/Makefile b/Makefile
index b6a62765f..f8b725900 100644
--- a/Makefile
+++ b/Makefile
@@ -2,6 +2,8 @@ default: build
 
 PACKAGES=./libs/... ./internal/... ./cmd/... ./bundle/... .
 
+GOTESTSUM_FORMAT ?= pkgname-and-test-fails
+
 lint:
 	./lint.sh ./...
 
@@ -9,10 +11,10 @@ lintcheck:
 	golangci-lint run ./...
 
 test:
-	gotestsum --format pkgname-and-test-fails --no-summary=skipped -- ${PACKAGES}
+	gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped -- ${PACKAGES}
 
 cover:
-	gotestsum --format pkgname-and-test-fails --no-summary=skipped -- -coverprofile=coverage.txt ${PACKAGES}
+	gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped -- -coverprofile=coverage.txt ${PACKAGES}
 
 showcover:
 	go tool cover -html=coverage.txt

From 31552852ff6856a70f54454322d7407f6fdcfb06 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Mon, 6 Jan 2025 15:30:48 +0100
Subject: [PATCH 026/247] Disable integration workflows (#2085)

See https://github.com/databricks/cli/issues/2084.
---
 .github/workflows/external-message.yml    | 12 ++++++++----
 .github/workflows/integration-approve.yml |  6 +++++-
 .github/workflows/integration-main.yml    | 10 +++++++---
 .github/workflows/integration-pr.yml      |  8 ++++++--
 4 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/external-message.yml b/.github/workflows/external-message.yml
index 28e61503c..9c91242a7 100644
--- a/.github/workflows/external-message.yml
+++ b/.github/workflows/external-message.yml
@@ -6,10 +6,14 @@ name: PR Comment
 # DO NOT PULL THE PR OR EXECUTE ANY CODE FROM THE PR.
 
 on:
-  pull_request_target:
-    types: [opened, reopened, synchronize]
-    branches:
-      - main
+  workflow_dispatch:
+
+  # Disable because of https://github.com/databricks/cli/issues/2084.
+  #
+  # pull_request_target:
+  #   types: [opened, reopened, synchronize]
+  #   branches:
+  #     - main
 
 jobs:
   comment-on-pr:
diff --git a/.github/workflows/integration-approve.yml b/.github/workflows/integration-approve.yml
index 4bdeb62a3..265574bb5 100644
--- a/.github/workflows/integration-approve.yml
+++ b/.github/workflows/integration-approve.yml
@@ -1,7 +1,11 @@
 name: integration-approve
 
 on:
-  merge_group:
+  workflow_dispatch:
+
+  # Disable because of https://github.com/databricks/cli/issues/2084.
+  #
+  # merge_group:
 
 jobs:
   # Trigger for merge groups.
diff --git a/.github/workflows/integration-main.yml b/.github/workflows/integration-main.yml
index 064e439cf..55fc98ae4 100644
--- a/.github/workflows/integration-main.yml
+++ b/.github/workflows/integration-main.yml
@@ -1,9 +1,13 @@
 name: integration-main
 
 on:
-  push:
-    branches:
-      - main
+  workflow_dispatch:
+
+  # Disable because of https://github.com/databricks/cli/issues/2084.
+  #
+  # push:
+  #   branches:
+  #     - main
 
 jobs:
   # Trigger for pushes to the main branch.
diff --git a/.github/workflows/integration-pr.yml b/.github/workflows/integration-pr.yml
index c1e3a9a29..965d74fbe 100644
--- a/.github/workflows/integration-pr.yml
+++ b/.github/workflows/integration-pr.yml
@@ -1,8 +1,12 @@
 name: integration-pr
 
 on:
-  pull_request:
-    types: [opened, synchronize]
+  workflow_dispatch:
+
+  # Disable because of https://github.com/databricks/cli/issues/2084.
+  #
+  # pull_request:
+  #   types: [opened, synchronize]
 
 jobs:
   # Trigger for pull requests.

From c262b30ef4a28685ef5e52c9d4985d0fe0591382 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Mon, 6 Jan 2025 16:34:42 +0100
Subject: [PATCH 027/247] Migrate workflows that need write access to use
 hosted runners (#2077)

## Changes

Migrate workflows to Databricks-hosted GitHub Actions runners.

The GitHub-hosted runners can no longer be used because of security
hardening.
---
 .github/workflows/close-stale-issues.yml  | 8 ++++++--
 .github/workflows/external-message.yml    | 5 ++++-
 .github/workflows/integration-approve.yml | 4 +++-
 .github/workflows/integration-main.yml    | 5 ++++-
 .github/workflows/integration-pr.yml      | 5 ++++-
 .github/workflows/release-snapshot.yml    | 5 ++++-
 .github/workflows/release.yml             | 6 +++++-
 7 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/close-stale-issues.yml b/.github/workflows/close-stale-issues.yml
index ffe550132..273b89a9c 100644
--- a/.github/workflows/close-stale-issues.yml
+++ b/.github/workflows/close-stale-issues.yml
@@ -7,12 +7,16 @@ on:
 
 jobs:
   cleanup:
+    name: Stale issue job
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     permissions:
       issues: write
       contents: read
       pull-requests: write
-    runs-on: ubuntu-latest
-    name: Stale issue job
+
     steps:
       - uses: actions/stale@v9
         with:
diff --git a/.github/workflows/external-message.yml b/.github/workflows/external-message.yml
index 9c91242a7..eb68a36e4 100644
--- a/.github/workflows/external-message.yml
+++ b/.github/workflows/external-message.yml
@@ -17,7 +17,10 @@ on:
 
 jobs:
   comment-on-pr:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     permissions:
       pull-requests: write
 
diff --git a/.github/workflows/integration-approve.yml b/.github/workflows/integration-approve.yml
index 265574bb5..0f6b209cb 100644
--- a/.github/workflows/integration-approve.yml
+++ b/.github/workflows/integration-approve.yml
@@ -21,7 +21,9 @@ jobs:
   #   * Avoid running integration tests twice, since it was already run at the tip of the branch before squashing.
   #
   trigger:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
 
     steps:
       - name: Auto-approve squashed commit
diff --git a/.github/workflows/integration-main.yml b/.github/workflows/integration-main.yml
index 55fc98ae4..5a78d4fd8 100644
--- a/.github/workflows/integration-main.yml
+++ b/.github/workflows/integration-main.yml
@@ -15,7 +15,10 @@ jobs:
   # This workflow triggers the integration test workflow in a different repository.
   # It requires secrets from the "test-trigger-is" environment, which are only available to authorized users.
   trigger:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     environment: "test-trigger-is"
 
     steps:
diff --git a/.github/workflows/integration-pr.yml b/.github/workflows/integration-pr.yml
index 965d74fbe..fd170f77e 100644
--- a/.github/workflows/integration-pr.yml
+++ b/.github/workflows/integration-pr.yml
@@ -14,7 +14,10 @@ jobs:
   # This workflow triggers the integration test workflow in a different repository.
   # It requires secrets from the "test-trigger-is" environment, which are only available to authorized users.
   trigger:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     environment: "test-trigger-is"
 
     # Only run this job for PRs from branches on the main repository and not from forks.
diff --git a/.github/workflows/release-snapshot.yml b/.github/workflows/release-snapshot.yml
index 7ef8b43c9..5c56a294e 100644
--- a/.github/workflows/release-snapshot.yml
+++ b/.github/workflows/release-snapshot.yml
@@ -20,7 +20,10 @@ on:
 
 jobs:
   goreleaser:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     steps:
       - name: Checkout repository and submodules
         uses: actions/checkout@v4
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index e4a253531..88e338a8c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -9,9 +9,13 @@ on:
 
 jobs:
   goreleaser:
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     outputs:
       artifacts: ${{ steps.releaser.outputs.artifacts }}
-    runs-on: ubuntu-latest
+
     steps:
       - name: Checkout repository and submodules
         uses: actions/checkout@v4

From 3629c9e4062f0de58c7c8ce9524e050621be3cb7 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Mon, 6 Jan 2025 17:07:15 +0100
Subject: [PATCH 028/247] Enable integration workflows (#2086)

## Changes

This reverts commit 31552852ff6856a70f54454322d7407f6fdcfb06.

These workflows were disabled in #2085.

They should work again now that we're using self-hosted runners (see
#2077).

## Tests

(inline)
---
 .github/workflows/external-message.yml    | 12 ++++--------
 .github/workflows/integration-approve.yml |  6 +-----
 .github/workflows/integration-main.yml    | 10 +++-------
 .github/workflows/integration-pr.yml      |  8 ++------
 4 files changed, 10 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/external-message.yml b/.github/workflows/external-message.yml
index eb68a36e4..f06d81a47 100644
--- a/.github/workflows/external-message.yml
+++ b/.github/workflows/external-message.yml
@@ -6,14 +6,10 @@ name: PR Comment
 # DO NOT PULL THE PR OR EXECUTE ANY CODE FROM THE PR.
 
 on:
-  workflow_dispatch:
-
-  # Disable because of https://github.com/databricks/cli/issues/2084.
-  #
-  # pull_request_target:
-  #   types: [opened, reopened, synchronize]
-  #   branches:
-  #     - main
+  pull_request_target:
+    types: [opened, reopened, synchronize]
+    branches:
+      - main
 
 jobs:
   comment-on-pr:
diff --git a/.github/workflows/integration-approve.yml b/.github/workflows/integration-approve.yml
index 0f6b209cb..293d31a2a 100644
--- a/.github/workflows/integration-approve.yml
+++ b/.github/workflows/integration-approve.yml
@@ -1,11 +1,7 @@
 name: integration-approve
 
 on:
-  workflow_dispatch:
-
-  # Disable because of https://github.com/databricks/cli/issues/2084.
-  #
-  # merge_group:
+  merge_group:
 
 jobs:
   # Trigger for merge groups.
diff --git a/.github/workflows/integration-main.yml b/.github/workflows/integration-main.yml
index 5a78d4fd8..0b6032d50 100644
--- a/.github/workflows/integration-main.yml
+++ b/.github/workflows/integration-main.yml
@@ -1,13 +1,9 @@
 name: integration-main
 
 on:
-  workflow_dispatch:
-
-  # Disable because of https://github.com/databricks/cli/issues/2084.
-  #
-  # push:
-  #   branches:
-  #     - main
+  push:
+    branches:
+      - main
 
 jobs:
   # Trigger for pushes to the main branch.
diff --git a/.github/workflows/integration-pr.yml b/.github/workflows/integration-pr.yml
index fd170f77e..0f9c4797a 100644
--- a/.github/workflows/integration-pr.yml
+++ b/.github/workflows/integration-pr.yml
@@ -1,12 +1,8 @@
 name: integration-pr
 
 on:
-  workflow_dispatch:
-
-  # Disable because of https://github.com/databricks/cli/issues/2084.
-  #
-  # pull_request:
-  #   types: [opened, synchronize]
+  pull_request:
+    types: [opened, synchronize]
 
 jobs:
   # Trigger for pull requests.

From e2cd8c2f34057c9d1f954f454d1a4d29bf656deb Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 7 Jan 2025 11:49:23 +0100
Subject: [PATCH 029/247] Enable perfsprint linter and apply autofix (#2071)

https://github.com/catenacyber/perfsprint
---
 .golangci.yaml                                |  1 +
 bundle/artifacts/expand_globs_test.go         |  9 ++++----
 bundle/artifacts/whl/infer.go                 |  2 +-
 bundle/bundle.go                              |  3 ++-
 bundle/config/artifact.go                     |  4 ++--
 .../config/mutator/expand_workspace_root.go   |  3 +--
 .../mutator/prepend_workspace_prefix.go       |  2 +-
 .../mutator/python/python_mutator_test.go     | 23 ++++++++++---------
 .../mutator/resolve_variable_references.go    |  6 ++---
 bundle/config/resources/clusters.go           |  3 +--
 bundle/config/resources/job.go                |  3 +--
 bundle/config/resources/mlflow_experiment.go  |  3 +--
 bundle/config/resources/mlflow_model.go       |  3 +--
 .../resources/model_serving_endpoint.go       |  3 +--
 bundle/config/resources/permission.go         |  2 +-
 bundle/config/resources/pipeline.go           |  3 +--
 bundle/config/resources/quality_monitor.go    |  3 +--
 bundle/config/resources/registered_model.go   |  3 +--
 bundle/config/resources/schema.go             |  6 ++---
 bundle/config/resources/volume.go             |  6 ++---
 bundle/config/validate/folder_permissions.go  |  3 ++-
 .../config/validate/unique_resource_keys.go   |  3 +--
 .../config/validate/validate_artifact_path.go |  2 +-
 .../validate/validate_artifact_path_test.go   |  5 ++--
 bundle/config/variable/lookup.go              |  6 ++---
 bundle/config/variable/resolve_alert.go       |  5 ++--
 bundle/config/variable/resolve_cluster.go     |  2 +-
 .../config/variable/resolve_cluster_policy.go |  5 ++--
 bundle/config/variable/resolve_dashboard.go   |  5 ++--
 .../config/variable/resolve_instance_pool.go  |  5 ++--
 bundle/config/variable/resolve_job.go         |  6 ++---
 bundle/config/variable/resolve_metastore.go   |  5 ++--
 .../resolve_notification_destination.go       |  2 +-
 .../resolve_notification_destination_test.go  |  4 ++--
 bundle/config/variable/resolve_pipeline.go    |  5 ++--
 bundle/config/variable/resolve_query.go       |  5 ++--
 .../variable/resolve_service_principal.go     |  5 ++--
 bundle/config/variable/resolve_warehouse.go   |  5 ++--
 bundle/config/variable/variable.go            |  3 ++-
 bundle/deploy/state.go                        |  3 ++-
 .../check_dashboards_modified_remotely.go     |  2 +-
 ...check_dashboards_modified_remotely_test.go |  4 ++--
 bundle/deploy/terraform/init.go               |  2 +-
 bundle/deploy/terraform/load.go               |  3 ++-
 bundle/libraries/helpers.go                   |  4 ++--
 bundle/permissions/workspace_root.go          |  3 ++-
 bundle/phases/deploy.go                       |  6 ++---
 bundle/phases/destroy.go                      |  5 ++--
 bundle/root.go                                |  3 ++-
 bundle/run/job.go                             |  7 +++---
 bundle/run/job_options.go                     | 10 ++++----
 bundle/run/output/job.go                      |  2 +-
 bundle/run/output/task.go                     |  9 ++++----
 bundle/run/pipeline.go                        |  9 ++++----
 bundle/run/progress/pipeline.go               |  2 +-
 bundle/tests/run_as_test.go                   |  3 +--
 bundle/trampoline/python_wheel.go             |  3 ++-
 bundle/trampoline/trampoline_test.go          |  4 ++--
 cmd/auth/auth.go                              |  6 ++---
 cmd/auth/describe_test.go                     |  6 ++---
 cmd/auth/env.go                               |  4 ++--
 cmd/auth/login.go                             |  2 +-
 cmd/bundle/destroy.go                         |  8 +++----
 cmd/bundle/generate/dashboard.go              |  6 ++---
 cmd/bundle/generate/job.go                    |  6 ++---
 cmd/bundle/generate/pipeline.go               |  6 ++---
 cmd/bundle/generate/utils.go                  |  2 +-
 cmd/bundle/launch.go                          |  4 ++--
 cmd/bundle/open.go                            |  4 ++--
 cmd/bundle/run.go                             |  3 ++-
 cmd/bundle/test.go                            |  4 ++--
 cmd/bundle/validate.go                        |  3 ++-
 cmd/configure/configure.go                    |  5 ++--
 cmd/configure/host.go                         |  6 ++---
 cmd/labs/github/repositories.go               |  2 +-
 cmd/labs/installed.go                         |  3 ++-
 cmd/labs/localcache/jsonfile.go               |  2 +-
 cmd/labs/localcache/jsonfile_test.go          |  3 +--
 cmd/labs/project/installer.go                 | 12 +++++-----
 cmd/labs/show.go                              |  4 ++--
 cmd/root/auth.go                              |  8 +++----
 cmd/root/progress_logger.go                   |  4 ++--
 cmd/sync/completion.go                        |  5 ++--
 cmd/sync/sync.go                              |  3 ++-
 cmd/workspace/repos/overrides.go              |  3 ++-
 cmd/workspace/secrets/put_secret.go           |  4 ++--
 cmd/workspace/workspace/overrides.go          |  2 +-
 integration/bundle/bind_resource_test.go      | 16 ++++++-------
 integration/bundle/clusters_test.go           |  5 ++--
 integration/bundle/dashboards_test.go         |  2 +-
 integration/bundle/deploy_to_shared_test.go   |  3 +--
 integration/bundle/empty_bundle_test.go       |  4 +---
 integration/bundle/generate_job_test.go       |  6 ++---
 integration/bundle/generate_pipeline_test.go  |  7 +++---
 integration/bundle/init_test.go               |  2 +-
 integration/cmd/auth/describe_test.go         | 11 ++++-----
 integration/cmd/fs/completion_test.go         |  3 +--
 integration/cmd/jobs/jobs_test.go             |  4 ++--
 integration/cmd/sync/sync_test.go             |  5 +---
 integration/cmd/workspace/workspace_test.go   |  4 ++--
 integration/internal/acc/fixtures.go          |  2 +-
 integration/python/python_tasks_test.go       | 11 ++++-----
 libs/auth/oauth.go                            | 12 +++++-----
 libs/auth/oauth_test.go                       |  4 ++--
 libs/cmdio/error_event.go                     |  4 +---
 libs/cmdio/logger.go                          |  7 +++---
 libs/databrickscfg/cfgpickers/clusters.go     |  2 +-
 libs/databrickscfg/loader.go                  |  2 +-
 libs/dyn/convert/normalize.go                 |  2 +-
 libs/dyn/jsonloader/json.go                   |  5 ++--
 libs/dyn/location.go                          |  3 ++-
 libs/dyn/mapping_test.go                      |  6 ++---
 libs/dyn/merge/override_test.go               |  4 ++--
 libs/dyn/path_string_test.go                  | 18 +++++++--------
 libs/dyn/visit_map_test.go                    |  9 ++++----
 libs/dyn/yamlsaver/saver.go                   |  4 ++--
 libs/errs/aggregate_test.go                   | 12 +++++-----
 libs/exec/shell_cmd.go                        |  3 +--
 libs/fakefs/fakefs.go                         |  4 ++--
 libs/filer/fake_filer.go                      |  8 +++----
 libs/filer/filer.go                           | 15 ++++++------
 libs/filer/files_client.go                    |  5 +---
 libs/filer/workspace_files_cache_test.go      |  4 ++--
 libs/flags/json_flag_test.go                  |  3 +--
 libs/flags/output.go                          |  3 ++-
 libs/git/reference.go                         |  4 ++--
 libs/git/repository_test.go                   |  4 +---
 libs/git/worktree_test.go                     |  6 ++---
 libs/jsonschema/instance.go                   |  5 ++--
 libs/jsonschema/utils.go                      |  2 +-
 libs/locker/locker.go                         |  6 ++---
 libs/process/background_test.go               |  4 ++--
 libs/process/stub.go                          |  2 +-
 libs/process/stub_test.go                     |  8 +++----
 libs/sync/event.go                            | 10 ++++----
 libs/sync/path.go                             |  2 +-
 libs/sync/sync.go                             |  3 ++-
 libs/tags/tag.go                              |  7 +++---
 libs/template/config.go                       |  4 ++--
 libs/template/materialize_test.go             |  3 +--
 libs/template/renderer_test.go                |  3 +--
 141 files changed, 330 insertions(+), 353 deletions(-)

diff --git a/.golangci.yaml b/.golangci.yaml
index 06d8152e5..07a6afdc5 100644
--- a/.golangci.yaml
+++ b/.golangci.yaml
@@ -14,6 +14,7 @@ linters:
     - testifylint
     - intrange
     - mirror
+    - perfsprint
 linters-settings:
   govet:
     enable-all: true
diff --git a/bundle/artifacts/expand_globs_test.go b/bundle/artifacts/expand_globs_test.go
index dc7c77de7..264c52c50 100644
--- a/bundle/artifacts/expand_globs_test.go
+++ b/bundle/artifacts/expand_globs_test.go
@@ -2,7 +2,6 @@ package artifacts
 
 import (
 	"context"
-	"fmt"
 	"path/filepath"
 	"testing"
 
@@ -88,16 +87,16 @@ func TestExpandGlobs_InvalidPattern(t *testing.T) {
 	))
 
 	assert.Len(t, diags, 4)
-	assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("a[.txt")), diags[0].Summary)
+	assert.Equal(t, filepath.Clean("a[.txt")+": syntax error in pattern", diags[0].Summary)
 	assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[0].Locations[0].File)
 	assert.Equal(t, "artifacts.test.files[0].source", diags[0].Paths[0].String())
-	assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("a[.txt")), diags[1].Summary)
+	assert.Equal(t, filepath.Clean("a[.txt")+": syntax error in pattern", diags[1].Summary)
 	assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[1].Locations[0].File)
 	assert.Equal(t, "artifacts.test.files[1].source", diags[1].Paths[0].String())
-	assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("../a[.txt")), diags[2].Summary)
+	assert.Equal(t, filepath.Clean("../a[.txt")+": syntax error in pattern", diags[2].Summary)
 	assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[2].Locations[0].File)
 	assert.Equal(t, "artifacts.test.files[2].source", diags[2].Paths[0].String())
-	assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("subdir/a[.txt")), diags[3].Summary)
+	assert.Equal(t, filepath.Clean("subdir/a[.txt")+": syntax error in pattern", diags[3].Summary)
 	assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[3].Locations[0].File)
 	assert.Equal(t, "artifacts.test.files[3].source", diags[3].Paths[0].String())
 }
diff --git a/bundle/artifacts/whl/infer.go b/bundle/artifacts/whl/infer.go
index 604bfc449..9c40360be 100644
--- a/bundle/artifacts/whl/infer.go
+++ b/bundle/artifacts/whl/infer.go
@@ -32,7 +32,7 @@ func (m *infer) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	//)
 
 	py := python.GetExecutable()
-	artifact.BuildCommand = fmt.Sprintf(`%s setup.py bdist_wheel`, py)
+	artifact.BuildCommand = py + " setup.py bdist_wheel"
 
 	return nil
 }
diff --git a/bundle/bundle.go b/bundle/bundle.go
index 573bcef2f..1f5e2a294 100644
--- a/bundle/bundle.go
+++ b/bundle/bundle.go
@@ -8,6 +8,7 @@ package bundle
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -234,7 +235,7 @@ func (b *Bundle) GetSyncIncludePatterns(ctx context.Context) ([]string, error) {
 // we call into from this bundle context.
 func (b *Bundle) AuthEnv() (map[string]string, error) {
 	if b.client == nil {
-		return nil, fmt.Errorf("workspace client not initialized yet")
+		return nil, errors.New("workspace client not initialized yet")
 	}
 
 	cfg := b.client.Config
diff --git a/bundle/config/artifact.go b/bundle/config/artifact.go
index 9a5690f57..177799e11 100644
--- a/bundle/config/artifact.go
+++ b/bundle/config/artifact.go
@@ -2,7 +2,7 @@ package config
 
 import (
 	"context"
-	"fmt"
+	"errors"
 
 	"github.com/databricks/cli/libs/exec"
 )
@@ -37,7 +37,7 @@ type Artifact struct {
 
 func (a *Artifact) Build(ctx context.Context) ([]byte, error) {
 	if a.BuildCommand == "" {
-		return nil, fmt.Errorf("no build property defined")
+		return nil, errors.New("no build property defined")
 	}
 
 	var e *exec.Executor
diff --git a/bundle/config/mutator/expand_workspace_root.go b/bundle/config/mutator/expand_workspace_root.go
index a29d129b0..2ec70548f 100644
--- a/bundle/config/mutator/expand_workspace_root.go
+++ b/bundle/config/mutator/expand_workspace_root.go
@@ -2,7 +2,6 @@ package mutator
 
 import (
 	"context"
-	"fmt"
 	"path"
 	"strings"
 
@@ -33,7 +32,7 @@ func (m *expandWorkspaceRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.
 	}
 
 	if strings.HasPrefix(root, "~/") {
-		home := fmt.Sprintf("/Workspace/Users/%s", currentUser.UserName)
+		home := "/Workspace/Users/" + currentUser.UserName
 		b.Config.Workspace.RootPath = path.Join(home, root[2:])
 	}
 
diff --git a/bundle/config/mutator/prepend_workspace_prefix.go b/bundle/config/mutator/prepend_workspace_prefix.go
index b093ec26a..616759ee4 100644
--- a/bundle/config/mutator/prepend_workspace_prefix.go
+++ b/bundle/config/mutator/prepend_workspace_prefix.go
@@ -55,7 +55,7 @@ func (m *prependWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) di
 					}
 				}
 
-				return dyn.NewValue(fmt.Sprintf("/Workspace%s", path), v.Locations()), nil
+				return dyn.NewValue("/Workspace"+path, v.Locations()), nil
 			})
 			if err != nil {
 				return dyn.InvalidValue, err
diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go
index 8bdf91d03..ff21f8ed9 100644
--- a/bundle/config/mutator/python/python_mutator_test.go
+++ b/bundle/config/mutator/python/python_mutator_test.go
@@ -2,6 +2,7 @@ package python
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
@@ -319,15 +320,15 @@ func TestCreateOverrideVisitor(t *testing.T) {
 			updatePath:  dyn.MustPathFromString("resources.jobs.job0.name"),
 			deletePath:  dyn.MustPathFromString("resources.jobs.job0.name"),
 			insertPath:  dyn.MustPathFromString("resources.jobs.job0.name"),
-			deleteError: fmt.Errorf("unexpected change at \"resources.jobs.job0.name\" (delete)"),
-			insertError: fmt.Errorf("unexpected change at \"resources.jobs.job0.name\" (insert)"),
-			updateError: fmt.Errorf("unexpected change at \"resources.jobs.job0.name\" (update)"),
+			deleteError: errors.New("unexpected change at \"resources.jobs.job0.name\" (delete)"),
+			insertError: errors.New("unexpected change at \"resources.jobs.job0.name\" (insert)"),
+			updateError: errors.New("unexpected change at \"resources.jobs.job0.name\" (update)"),
 		},
 		{
 			name:        "load: can't delete an existing job",
 			phase:       PythonMutatorPhaseLoad,
 			deletePath:  dyn.MustPathFromString("resources.jobs.job0"),
-			deleteError: fmt.Errorf("unexpected change at \"resources.jobs.job0\" (delete)"),
+			deleteError: errors.New("unexpected change at \"resources.jobs.job0\" (delete)"),
 		},
 		{
 			name:        "load: can insert 'resources'",
@@ -353,9 +354,9 @@ func TestCreateOverrideVisitor(t *testing.T) {
 			deletePath:  dyn.MustPathFromString("include[0]"),
 			insertPath:  dyn.MustPathFromString("include[0]"),
 			updatePath:  dyn.MustPathFromString("include[0]"),
-			deleteError: fmt.Errorf("unexpected change at \"include[0]\" (delete)"),
-			insertError: fmt.Errorf("unexpected change at \"include[0]\" (insert)"),
-			updateError: fmt.Errorf("unexpected change at \"include[0]\" (update)"),
+			deleteError: errors.New("unexpected change at \"include[0]\" (delete)"),
+			insertError: errors.New("unexpected change at \"include[0]\" (insert)"),
+			updateError: errors.New("unexpected change at \"include[0]\" (update)"),
 		},
 		{
 			name:        "init: can change an existing job",
@@ -371,7 +372,7 @@ func TestCreateOverrideVisitor(t *testing.T) {
 			name:        "init: can't delete an existing job",
 			phase:       PythonMutatorPhaseInit,
 			deletePath:  dyn.MustPathFromString("resources.jobs.job0"),
-			deleteError: fmt.Errorf("unexpected change at \"resources.jobs.job0\" (delete)"),
+			deleteError: errors.New("unexpected change at \"resources.jobs.job0\" (delete)"),
 		},
 		{
 			name:        "init: can insert 'resources'",
@@ -397,9 +398,9 @@ func TestCreateOverrideVisitor(t *testing.T) {
 			deletePath:  dyn.MustPathFromString("include[0]"),
 			insertPath:  dyn.MustPathFromString("include[0]"),
 			updatePath:  dyn.MustPathFromString("include[0]"),
-			deleteError: fmt.Errorf("unexpected change at \"include[0]\" (delete)"),
-			insertError: fmt.Errorf("unexpected change at \"include[0]\" (insert)"),
-			updateError: fmt.Errorf("unexpected change at \"include[0]\" (update)"),
+			deleteError: errors.New("unexpected change at \"include[0]\" (delete)"),
+			insertError: errors.New("unexpected change at \"include[0]\" (insert)"),
+			updateError: errors.New("unexpected change at \"include[0]\" (update)"),
 		},
 	}
 
diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go
index 8c207e375..e074c2b84 100644
--- a/bundle/config/mutator/resolve_variable_references.go
+++ b/bundle/config/mutator/resolve_variable_references.go
@@ -2,7 +2,7 @@ package mutator
 
 import (
 	"context"
-	"fmt"
+	"errors"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config/variable"
@@ -68,7 +68,7 @@ func lookupForComplexVariables(v dyn.Value, path dyn.Path) (dyn.Value, error) {
 	}
 
 	if vv.Type == variable.VariableTypeComplex {
-		return dyn.InvalidValue, fmt.Errorf("complex variables cannot contain references to another complex variables")
+		return dyn.InvalidValue, errors.New("complex variables cannot contain references to another complex variables")
 	}
 
 	return lookup(v, path)
@@ -100,7 +100,7 @@ func lookupForVariables(v dyn.Value, path dyn.Path) (dyn.Value, error) {
 	}
 
 	if vv.Lookup != nil && vv.Lookup.String() != "" {
-		return dyn.InvalidValue, fmt.Errorf("lookup variables cannot contain references to another lookup variables")
+		return dyn.InvalidValue, errors.New("lookup variables cannot contain references to another lookup variables")
 	}
 
 	return lookup(v, path)
diff --git a/bundle/config/resources/clusters.go b/bundle/config/resources/clusters.go
index ba991e865..073f40a79 100644
--- a/bundle/config/resources/clusters.go
+++ b/bundle/config/resources/clusters.go
@@ -2,7 +2,6 @@ package resources
 
 import (
 	"context"
-	"fmt"
 	"net/url"
 
 	"github.com/databricks/cli/libs/log"
@@ -45,7 +44,7 @@ func (s *Cluster) InitializeURL(baseURL url.URL) {
 	if s.ID == "" {
 		return
 	}
-	baseURL.Path = fmt.Sprintf("compute/clusters/%s", s.ID)
+	baseURL.Path = "compute/clusters/" + s.ID
 	s.URL = baseURL.String()
 }
 
diff --git a/bundle/config/resources/job.go b/bundle/config/resources/job.go
index 0aa41b2e8..76de78439 100644
--- a/bundle/config/resources/job.go
+++ b/bundle/config/resources/job.go
@@ -2,7 +2,6 @@ package resources
 
 import (
 	"context"
-	"fmt"
 	"net/url"
 	"strconv"
 
@@ -52,7 +51,7 @@ func (j *Job) InitializeURL(baseURL url.URL) {
 	if j.ID == "" {
 		return
 	}
-	baseURL.Path = fmt.Sprintf("jobs/%s", j.ID)
+	baseURL.Path = "jobs/" + j.ID
 	j.URL = baseURL.String()
 }
 
diff --git a/bundle/config/resources/mlflow_experiment.go b/bundle/config/resources/mlflow_experiment.go
index 5d179ec0f..ea18ce114 100644
--- a/bundle/config/resources/mlflow_experiment.go
+++ b/bundle/config/resources/mlflow_experiment.go
@@ -2,7 +2,6 @@ package resources
 
 import (
 	"context"
-	"fmt"
 	"net/url"
 
 	"github.com/databricks/cli/libs/log"
@@ -47,7 +46,7 @@ func (s *MlflowExperiment) InitializeURL(baseURL url.URL) {
 	if s.ID == "" {
 		return
 	}
-	baseURL.Path = fmt.Sprintf("ml/experiments/%s", s.ID)
+	baseURL.Path = "ml/experiments/" + s.ID
 	s.URL = baseURL.String()
 }
 
diff --git a/bundle/config/resources/mlflow_model.go b/bundle/config/resources/mlflow_model.go
index 72376f45d..69ae2d438 100644
--- a/bundle/config/resources/mlflow_model.go
+++ b/bundle/config/resources/mlflow_model.go
@@ -2,7 +2,6 @@ package resources
 
 import (
 	"context"
-	"fmt"
 	"net/url"
 
 	"github.com/databricks/cli/libs/log"
@@ -47,7 +46,7 @@ func (s *MlflowModel) InitializeURL(baseURL url.URL) {
 	if s.ID == "" {
 		return
 	}
-	baseURL.Path = fmt.Sprintf("ml/models/%s", s.ID)
+	baseURL.Path = "ml/models/" + s.ID
 	s.URL = baseURL.String()
 }
 
diff --git a/bundle/config/resources/model_serving_endpoint.go b/bundle/config/resources/model_serving_endpoint.go
index a3c472b3f..8b1394d86 100644
--- a/bundle/config/resources/model_serving_endpoint.go
+++ b/bundle/config/resources/model_serving_endpoint.go
@@ -2,7 +2,6 @@ package resources
 
 import (
 	"context"
-	"fmt"
 	"net/url"
 
 	"github.com/databricks/cli/libs/log"
@@ -55,7 +54,7 @@ func (s *ModelServingEndpoint) InitializeURL(baseURL url.URL) {
 	if s.ID == "" {
 		return
 	}
-	baseURL.Path = fmt.Sprintf("ml/endpoints/%s", s.ID)
+	baseURL.Path = "ml/endpoints/" + s.ID
 	s.URL = baseURL.String()
 }
 
diff --git a/bundle/config/resources/permission.go b/bundle/config/resources/permission.go
index 62e18a09e..fa1568601 100644
--- a/bundle/config/resources/permission.go
+++ b/bundle/config/resources/permission.go
@@ -25,5 +25,5 @@ func (p Permission) String() string {
 		return fmt.Sprintf("level: %s, group_name: %s", p.Level, p.GroupName)
 	}
 
-	return fmt.Sprintf("level: %s", p.Level)
+	return "level: " + p.Level
 }
diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go
index eaa4c5368..5127d07ba 100644
--- a/bundle/config/resources/pipeline.go
+++ b/bundle/config/resources/pipeline.go
@@ -2,7 +2,6 @@ package resources
 
 import (
 	"context"
-	"fmt"
 	"net/url"
 
 	"github.com/databricks/cli/libs/log"
@@ -47,7 +46,7 @@ func (p *Pipeline) InitializeURL(baseURL url.URL) {
 	if p.ID == "" {
 		return
 	}
-	baseURL.Path = fmt.Sprintf("pipelines/%s", p.ID)
+	baseURL.Path = "pipelines/" + p.ID
 	p.URL = baseURL.String()
 }
 
diff --git a/bundle/config/resources/quality_monitor.go b/bundle/config/resources/quality_monitor.go
index b1d7e08a5..88bc0a3e7 100644
--- a/bundle/config/resources/quality_monitor.go
+++ b/bundle/config/resources/quality_monitor.go
@@ -2,7 +2,6 @@ package resources
 
 import (
 	"context"
-	"fmt"
 	"net/url"
 	"strings"
 
@@ -51,7 +50,7 @@ func (s *QualityMonitor) InitializeURL(baseURL url.URL) {
 	if s.TableName == "" {
 		return
 	}
-	baseURL.Path = fmt.Sprintf("explore/data/%s", strings.ReplaceAll(s.TableName, ".", "/"))
+	baseURL.Path = "explore/data/" + strings.ReplaceAll(s.TableName, ".", "/")
 	s.URL = baseURL.String()
 }
 
diff --git a/bundle/config/resources/registered_model.go b/bundle/config/resources/registered_model.go
index 8513a79ae..006eef773 100644
--- a/bundle/config/resources/registered_model.go
+++ b/bundle/config/resources/registered_model.go
@@ -2,7 +2,6 @@ package resources
 
 import (
 	"context"
-	"fmt"
 	"net/url"
 	"strings"
 
@@ -57,7 +56,7 @@ func (s *RegisteredModel) InitializeURL(baseURL url.URL) {
 	if s.ID == "" {
 		return
 	}
-	baseURL.Path = fmt.Sprintf("explore/data/models/%s", strings.ReplaceAll(s.ID, ".", "/"))
+	baseURL.Path = "explore/data/models/" + strings.ReplaceAll(s.ID, ".", "/")
 	s.URL = baseURL.String()
 }
 
diff --git a/bundle/config/resources/schema.go b/bundle/config/resources/schema.go
index 8eadd7e46..b638907ac 100644
--- a/bundle/config/resources/schema.go
+++ b/bundle/config/resources/schema.go
@@ -2,7 +2,7 @@ package resources
 
 import (
 	"context"
-	"fmt"
+	"errors"
 	"net/url"
 	"strings"
 
@@ -26,7 +26,7 @@ type Schema struct {
 }
 
 func (s *Schema) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
-	return false, fmt.Errorf("schema.Exists() is not supported")
+	return false, errors.New("schema.Exists() is not supported")
 }
 
 func (s *Schema) TerraformResourceName() string {
@@ -37,7 +37,7 @@ func (s *Schema) InitializeURL(baseURL url.URL) {
 	if s.ID == "" {
 		return
 	}
-	baseURL.Path = fmt.Sprintf("explore/data/%s", strings.ReplaceAll(s.ID, ".", "/"))
+	baseURL.Path = "explore/data/" + strings.ReplaceAll(s.ID, ".", "/")
 	s.URL = baseURL.String()
 }
 
diff --git a/bundle/config/resources/volume.go b/bundle/config/resources/volume.go
index cae2a3463..882b7107d 100644
--- a/bundle/config/resources/volume.go
+++ b/bundle/config/resources/volume.go
@@ -2,7 +2,7 @@ package resources
 
 import (
 	"context"
-	"fmt"
+	"errors"
 	"net/url"
 	"strings"
 
@@ -34,7 +34,7 @@ func (v Volume) MarshalJSON() ([]byte, error) {
 }
 
 func (v *Volume) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
-	return false, fmt.Errorf("volume.Exists() is not supported")
+	return false, errors.New("volume.Exists() is not supported")
 }
 
 func (v *Volume) TerraformResourceName() string {
@@ -45,7 +45,7 @@ func (v *Volume) InitializeURL(baseURL url.URL) {
 	if v.ID == "" {
 		return
 	}
-	baseURL.Path = fmt.Sprintf("explore/data/volumes/%s", strings.ReplaceAll(v.ID, ".", "/"))
+	baseURL.Path = "explore/data/volumes/" + strings.ReplaceAll(v.ID, ".", "/")
 	v.URL = baseURL.String()
 }
 
diff --git a/bundle/config/validate/folder_permissions.go b/bundle/config/validate/folder_permissions.go
index 5b28d599e..7b12b4b16 100644
--- a/bundle/config/validate/folder_permissions.go
+++ b/bundle/config/validate/folder_permissions.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"path"
+	"strconv"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/libraries"
@@ -60,7 +61,7 @@ func checkFolderPermission(ctx context.Context, b bundle.ReadOnlyBundle, folderP
 	}
 
 	objPermissions, err := w.GetPermissions(ctx, workspace.GetWorkspaceObjectPermissionsRequest{
-		WorkspaceObjectId:   fmt.Sprint(obj.ObjectId),
+		WorkspaceObjectId:   strconv.FormatInt(obj.ObjectId, 10),
 		WorkspaceObjectType: "directories",
 	})
 	if err != nil {
diff --git a/bundle/config/validate/unique_resource_keys.go b/bundle/config/validate/unique_resource_keys.go
index 50295375b..d80c5d632 100644
--- a/bundle/config/validate/unique_resource_keys.go
+++ b/bundle/config/validate/unique_resource_keys.go
@@ -2,7 +2,6 @@ package validate
 
 import (
 	"context"
-	"fmt"
 	"sort"
 
 	"github.com/databricks/cli/bundle"
@@ -102,7 +101,7 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D
 		// If there are multiple resources with the same key, report an error.
 		diags = append(diags, diag.Diagnostic{
 			Severity:  diag.Error,
-			Summary:   fmt.Sprintf("multiple resources have been defined with the same key: %s", k),
+			Summary:   "multiple resources have been defined with the same key: " + k,
 			Locations: v.locations,
 			Paths:     v.paths,
 		})
diff --git a/bundle/config/validate/validate_artifact_path.go b/bundle/config/validate/validate_artifact_path.go
index 5bab99ccf..aa4492670 100644
--- a/bundle/config/validate/validate_artifact_path.go
+++ b/bundle/config/validate/validate_artifact_path.go
@@ -68,7 +68,7 @@ func findVolumeInBundle(r config.Root, catalogName, schemaName, volumeName strin
 		if v.SchemaName != schemaName && !isSchemaDefinedInBundle {
 			continue
 		}
-		pathString := fmt.Sprintf("resources.volumes.%s", k)
+		pathString := "resources.volumes." + k
 		return dyn.MustPathFromString(pathString), r.GetLocations(pathString), true
 	}
 	return nil, nil, false
diff --git a/bundle/config/validate/validate_artifact_path_test.go b/bundle/config/validate/validate_artifact_path_test.go
index 8fb5c9618..e1ae6af34 100644
--- a/bundle/config/validate/validate_artifact_path_test.go
+++ b/bundle/config/validate/validate_artifact_path_test.go
@@ -2,7 +2,6 @@ package validate
 
 import (
 	"context"
-	"fmt"
 	"testing"
 
 	"github.com/databricks/cli/bundle"
@@ -152,7 +151,7 @@ func TestExtractVolumeFromPath(t *testing.T) {
 
 	for _, p := range invalidVolumePaths() {
 		_, _, _, err := extractVolumeFromPath(p)
-		assert.EqualError(t, err, fmt.Sprintf("expected UC volume path to be in the format /Volumes/<catalog>/<schema>/<volume>/..., got %s", p))
+		assert.EqualError(t, err, "expected UC volume path to be in the format /Volumes/<catalog>/<schema>/<volume>/..., got "+p)
 	}
 }
 
@@ -171,7 +170,7 @@ func TestValidateArtifactPathWithInvalidPaths(t *testing.T) {
 		diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), ValidateArtifactPath())
 		require.Equal(t, diag.Diagnostics{{
 			Severity:  diag.Error,
-			Summary:   fmt.Sprintf("expected UC volume path to be in the format /Volumes/<catalog>/<schema>/<volume>/..., got %s", p),
+			Summary:   "expected UC volume path to be in the format /Volumes/<catalog>/<schema>/<volume>/..., got " + p,
 			Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}},
 			Paths:     []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")},
 		}}, diags)
diff --git a/bundle/config/variable/lookup.go b/bundle/config/variable/lookup.go
index 37e380f18..71c8512e3 100755
--- a/bundle/config/variable/lookup.go
+++ b/bundle/config/variable/lookup.go
@@ -2,7 +2,7 @@ package variable
 
 import (
 	"context"
-	"fmt"
+	"errors"
 
 	"github.com/databricks/databricks-sdk-go"
 )
@@ -83,11 +83,11 @@ func (l *Lookup) constructResolver() (resolver, error) {
 
 	switch len(resolvers) {
 	case 0:
-		return nil, fmt.Errorf("no valid lookup fields provided")
+		return nil, errors.New("no valid lookup fields provided")
 	case 1:
 		return resolvers[0], nil
 	default:
-		return nil, fmt.Errorf("exactly one lookup field must be provided")
+		return nil, errors.New("exactly one lookup field must be provided")
 	}
 }
 
diff --git a/bundle/config/variable/resolve_alert.go b/bundle/config/variable/resolve_alert.go
index be83e81fa..507306aa0 100644
--- a/bundle/config/variable/resolve_alert.go
+++ b/bundle/config/variable/resolve_alert.go
@@ -2,7 +2,6 @@ package variable
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/databricks/databricks-sdk-go"
 )
@@ -16,9 +15,9 @@ func (l resolveAlert) Resolve(ctx context.Context, w *databricks.WorkspaceClient
 	if err != nil {
 		return "", err
 	}
-	return fmt.Sprint(entity.Id), nil
+	return entity.Id, nil
 }
 
 func (l resolveAlert) String() string {
-	return fmt.Sprintf("alert: %s", l.name)
+	return "alert: " + l.name
 }
diff --git a/bundle/config/variable/resolve_cluster.go b/bundle/config/variable/resolve_cluster.go
index a8cf3fe7f..51278aef5 100644
--- a/bundle/config/variable/resolve_cluster.go
+++ b/bundle/config/variable/resolve_cluster.go
@@ -42,5 +42,5 @@ func (l resolveCluster) Resolve(ctx context.Context, w *databricks.WorkspaceClie
 }
 
 func (l resolveCluster) String() string {
-	return fmt.Sprintf("cluster: %s", l.name)
+	return "cluster: " + l.name
 }
diff --git a/bundle/config/variable/resolve_cluster_policy.go b/bundle/config/variable/resolve_cluster_policy.go
index b19380a63..94fd892b2 100644
--- a/bundle/config/variable/resolve_cluster_policy.go
+++ b/bundle/config/variable/resolve_cluster_policy.go
@@ -2,7 +2,6 @@ package variable
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/databricks/databricks-sdk-go"
 )
@@ -16,9 +15,9 @@ func (l resolveClusterPolicy) Resolve(ctx context.Context, w *databricks.Workspa
 	if err != nil {
 		return "", err
 	}
-	return fmt.Sprint(entity.PolicyId), nil
+	return entity.PolicyId, nil
 }
 
 func (l resolveClusterPolicy) String() string {
-	return fmt.Sprintf("cluster-policy: %s", l.name)
+	return "cluster-policy: " + l.name
 }
diff --git a/bundle/config/variable/resolve_dashboard.go b/bundle/config/variable/resolve_dashboard.go
index 44fd45197..2979716ce 100644
--- a/bundle/config/variable/resolve_dashboard.go
+++ b/bundle/config/variable/resolve_dashboard.go
@@ -2,7 +2,6 @@ package variable
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/databricks/databricks-sdk-go"
 )
@@ -16,9 +15,9 @@ func (l resolveDashboard) Resolve(ctx context.Context, w *databricks.WorkspaceCl
 	if err != nil {
 		return "", err
 	}
-	return fmt.Sprint(entity.Id), nil
+	return entity.Id, nil
 }
 
 func (l resolveDashboard) String() string {
-	return fmt.Sprintf("dashboard: %s", l.name)
+	return "dashboard: " + l.name
 }
diff --git a/bundle/config/variable/resolve_instance_pool.go b/bundle/config/variable/resolve_instance_pool.go
index cbf0775c9..600b47a50 100644
--- a/bundle/config/variable/resolve_instance_pool.go
+++ b/bundle/config/variable/resolve_instance_pool.go
@@ -2,7 +2,6 @@ package variable
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/databricks/databricks-sdk-go"
 )
@@ -16,9 +15,9 @@ func (l resolveInstancePool) Resolve(ctx context.Context, w *databricks.Workspac
 	if err != nil {
 		return "", err
 	}
-	return fmt.Sprint(entity.InstancePoolId), nil
+	return entity.InstancePoolId, nil
 }
 
 func (l resolveInstancePool) String() string {
-	return fmt.Sprintf("instance-pool: %s", l.name)
+	return "instance-pool: " + l.name
 }
diff --git a/bundle/config/variable/resolve_job.go b/bundle/config/variable/resolve_job.go
index 3def64888..4fe6ae3e7 100644
--- a/bundle/config/variable/resolve_job.go
+++ b/bundle/config/variable/resolve_job.go
@@ -2,7 +2,7 @@ package variable
 
 import (
 	"context"
-	"fmt"
+	"strconv"
 
 	"github.com/databricks/databricks-sdk-go"
 )
@@ -16,9 +16,9 @@ func (l resolveJob) Resolve(ctx context.Context, w *databricks.WorkspaceClient)
 	if err != nil {
 		return "", err
 	}
-	return fmt.Sprint(entity.JobId), nil
+	return strconv.FormatInt(entity.JobId, 10), nil
 }
 
 func (l resolveJob) String() string {
-	return fmt.Sprintf("job: %s", l.name)
+	return "job: " + l.name
 }
diff --git a/bundle/config/variable/resolve_metastore.go b/bundle/config/variable/resolve_metastore.go
index 958e43787..8a0a8c7ed 100644
--- a/bundle/config/variable/resolve_metastore.go
+++ b/bundle/config/variable/resolve_metastore.go
@@ -2,7 +2,6 @@ package variable
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/databricks/databricks-sdk-go"
 )
@@ -16,9 +15,9 @@ func (l resolveMetastore) Resolve(ctx context.Context, w *databricks.WorkspaceCl
 	if err != nil {
 		return "", err
 	}
-	return fmt.Sprint(entity.MetastoreId), nil
+	return entity.MetastoreId, nil
 }
 
 func (l resolveMetastore) String() string {
-	return fmt.Sprintf("metastore: %s", l.name)
+	return "metastore: " + l.name
 }
diff --git a/bundle/config/variable/resolve_notification_destination.go b/bundle/config/variable/resolve_notification_destination.go
index 4c4cd892a..4696a52c8 100644
--- a/bundle/config/variable/resolve_notification_destination.go
+++ b/bundle/config/variable/resolve_notification_destination.go
@@ -42,5 +42,5 @@ func (l resolveNotificationDestination) Resolve(ctx context.Context, w *databric
 }
 
 func (l resolveNotificationDestination) String() string {
-	return fmt.Sprintf("notification-destination: %s", l.name)
+	return "notification-destination: " + l.name
 }
diff --git a/bundle/config/variable/resolve_notification_destination_test.go b/bundle/config/variable/resolve_notification_destination_test.go
index 2b8201d15..f44b2f3e9 100644
--- a/bundle/config/variable/resolve_notification_destination_test.go
+++ b/bundle/config/variable/resolve_notification_destination_test.go
@@ -2,7 +2,7 @@ package variable
 
 import (
 	"context"
-	"fmt"
+	"errors"
 	"testing"
 
 	"github.com/databricks/databricks-sdk-go/experimental/mocks"
@@ -35,7 +35,7 @@ func TestResolveNotificationDestination_ResolveError(t *testing.T) {
 	api := m.GetMockNotificationDestinationsAPI()
 	api.EXPECT().
 		ListAll(mock.Anything, mock.Anything).
-		Return(nil, fmt.Errorf("bad"))
+		Return(nil, errors.New("bad"))
 
 	ctx := context.Background()
 	l := resolveNotificationDestination{name: "destination"}
diff --git a/bundle/config/variable/resolve_pipeline.go b/bundle/config/variable/resolve_pipeline.go
index cabc620da..33b14530d 100644
--- a/bundle/config/variable/resolve_pipeline.go
+++ b/bundle/config/variable/resolve_pipeline.go
@@ -2,7 +2,6 @@ package variable
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/databricks/databricks-sdk-go"
 )
@@ -16,9 +15,9 @@ func (l resolvePipeline) Resolve(ctx context.Context, w *databricks.WorkspaceCli
 	if err != nil {
 		return "", err
 	}
-	return fmt.Sprint(entity.PipelineId), nil
+	return entity.PipelineId, nil
 }
 
 func (l resolvePipeline) String() string {
-	return fmt.Sprintf("pipeline: %s", l.name)
+	return "pipeline: " + l.name
 }
diff --git a/bundle/config/variable/resolve_query.go b/bundle/config/variable/resolve_query.go
index 602ff8deb..88f653dc6 100644
--- a/bundle/config/variable/resolve_query.go
+++ b/bundle/config/variable/resolve_query.go
@@ -2,7 +2,6 @@ package variable
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/databricks/databricks-sdk-go"
 )
@@ -16,9 +15,9 @@ func (l resolveQuery) Resolve(ctx context.Context, w *databricks.WorkspaceClient
 	if err != nil {
 		return "", err
 	}
-	return fmt.Sprint(entity.Id), nil
+	return entity.Id, nil
 }
 
 func (l resolveQuery) String() string {
-	return fmt.Sprintf("query: %s", l.name)
+	return "query: " + l.name
 }
diff --git a/bundle/config/variable/resolve_service_principal.go b/bundle/config/variable/resolve_service_principal.go
index 3bea4314b..03b8e3089 100644
--- a/bundle/config/variable/resolve_service_principal.go
+++ b/bundle/config/variable/resolve_service_principal.go
@@ -2,7 +2,6 @@ package variable
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/databricks/databricks-sdk-go"
 )
@@ -16,9 +15,9 @@ func (l resolveServicePrincipal) Resolve(ctx context.Context, w *databricks.Work
 	if err != nil {
 		return "", err
 	}
-	return fmt.Sprint(entity.ApplicationId), nil
+	return entity.ApplicationId, nil
 }
 
 func (l resolveServicePrincipal) String() string {
-	return fmt.Sprintf("service-principal: %s", l.name)
+	return "service-principal: " + l.name
 }
diff --git a/bundle/config/variable/resolve_warehouse.go b/bundle/config/variable/resolve_warehouse.go
index fbd3663a2..cabdb1160 100644
--- a/bundle/config/variable/resolve_warehouse.go
+++ b/bundle/config/variable/resolve_warehouse.go
@@ -2,7 +2,6 @@ package variable
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/databricks/databricks-sdk-go"
 )
@@ -16,9 +15,9 @@ func (l resolveWarehouse) Resolve(ctx context.Context, w *databricks.WorkspaceCl
 	if err != nil {
 		return "", err
 	}
-	return fmt.Sprint(entity.Id), nil
+	return entity.Id, nil
 }
 
 func (l resolveWarehouse) String() string {
-	return fmt.Sprintf("warehouse: %s", l.name)
+	return "warehouse: " + l.name
 }
diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go
index 2362ad10d..95a68cfeb 100644
--- a/bundle/config/variable/variable.go
+++ b/bundle/config/variable/variable.go
@@ -1,6 +1,7 @@
 package variable
 
 import (
+	"errors"
 	"fmt"
 	"reflect"
 )
@@ -68,7 +69,7 @@ func (v *Variable) Set(val VariableValue) error {
 	switch rv.Kind() {
 	case reflect.Struct, reflect.Array, reflect.Slice, reflect.Map:
 		if v.Type != VariableTypeComplex {
-			return fmt.Errorf("variable type is not complex")
+			return errors.New("variable type is not complex")
 		}
 	}
 
diff --git a/bundle/deploy/state.go b/bundle/deploy/state.go
index a131ab9c3..6e285034a 100644
--- a/bundle/deploy/state.go
+++ b/bundle/deploy/state.go
@@ -3,6 +3,7 @@ package deploy
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/fs"
@@ -95,7 +96,7 @@ func (e *entry) Type() fs.FileMode {
 
 func (e *entry) Info() (fs.FileInfo, error) {
 	if e.info == nil {
-		return nil, fmt.Errorf("no info available")
+		return nil, errors.New("no info available")
 	}
 	return e.info, nil
 }
diff --git a/bundle/deploy/terraform/check_dashboards_modified_remotely.go b/bundle/deploy/terraform/check_dashboards_modified_remotely.go
index f263e8a7f..66914af54 100644
--- a/bundle/deploy/terraform/check_dashboards_modified_remotely.go
+++ b/bundle/deploy/terraform/check_dashboards_modified_remotely.go
@@ -72,7 +72,7 @@ func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.B
 			continue
 		}
 
-		path := dyn.MustPathFromString(fmt.Sprintf("resources.dashboards.%s", dashboard.Name))
+		path := dyn.MustPathFromString("resources.dashboards." + dashboard.Name)
 		loc := b.Config.GetLocation(path.String())
 		actual, err := b.WorkspaceClient().Lakeview.GetByDashboardId(ctx, dashboard.ID)
 		if err != nil {
diff --git a/bundle/deploy/terraform/check_dashboards_modified_remotely_test.go b/bundle/deploy/terraform/check_dashboards_modified_remotely_test.go
index 1bed3b1be..46bdc1f38 100644
--- a/bundle/deploy/terraform/check_dashboards_modified_remotely_test.go
+++ b/bundle/deploy/terraform/check_dashboards_modified_remotely_test.go
@@ -2,7 +2,7 @@ package terraform
 
 import (
 	"context"
-	"fmt"
+	"errors"
 	"path/filepath"
 	"testing"
 
@@ -122,7 +122,7 @@ func TestCheckDashboardsModifiedRemotely_ExistingStateFailureToGet(t *testing.T)
 	dashboardsAPI := m.GetMockLakeviewAPI()
 	dashboardsAPI.EXPECT().
 		GetByDashboardId(mock.Anything, "id1").
-		Return(nil, fmt.Errorf("failure")).
+		Return(nil, errors.New("failure")).
 		Once()
 	b.SetWorkpaceClient(m.WorkspaceClient)
 
diff --git a/bundle/deploy/terraform/init.go b/bundle/deploy/terraform/init.go
index 366f0be8c..e69f0bf0f 100644
--- a/bundle/deploy/terraform/init.go
+++ b/bundle/deploy/terraform/init.go
@@ -230,7 +230,7 @@ func setUserAgentExtraEnvVar(environ map[string]string, b *bundle.Bundle) error
 	// Add "cli" to the user agent in set by the Databricks Terraform provider.
 	// This will allow us to attribute downstream requests made by the Databricks
 	// Terraform provider to the CLI.
-	products := []string{fmt.Sprintf("cli/%s", build.GetInfo().Version)}
+	products := []string{"cli/" + build.GetInfo().Version}
 	if experimental := b.Config.Experimental; experimental != nil {
 		if experimental.PyDABs.Enabled {
 			products = append(products, "databricks-pydabs/0.0.0")
diff --git a/bundle/deploy/terraform/load.go b/bundle/deploy/terraform/load.go
index 3fb76855e..1c563fa77 100644
--- a/bundle/deploy/terraform/load.go
+++ b/bundle/deploy/terraform/load.go
@@ -2,6 +2,7 @@ package terraform
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"slices"
 
@@ -58,7 +59,7 @@ func (l *load) validateState(state *resourcesState) error {
 	}
 
 	if len(state.Resources) == 0 && slices.Contains(l.modes, ErrorOnEmptyState) {
-		return fmt.Errorf("no deployment state. Did you forget to run 'databricks bundle deploy'?")
+		return errors.New("no deployment state. Did you forget to run 'databricks bundle deploy'?")
 	}
 
 	return nil
diff --git a/bundle/libraries/helpers.go b/bundle/libraries/helpers.go
index 2149e5885..5a1a9511c 100644
--- a/bundle/libraries/helpers.go
+++ b/bundle/libraries/helpers.go
@@ -1,7 +1,7 @@
 package libraries
 
 import (
-	"fmt"
+	"errors"
 
 	"github.com/databricks/databricks-sdk-go/service/compute"
 )
@@ -20,5 +20,5 @@ func libraryPath(library *compute.Library) (string, error) {
 		return library.Requirements, nil
 	}
 
-	return "", fmt.Errorf("not supported library type")
+	return "", errors.New("not supported library type")
 }
diff --git a/bundle/permissions/workspace_root.go b/bundle/permissions/workspace_root.go
index 4ac0d38a5..828b12f50 100644
--- a/bundle/permissions/workspace_root.go
+++ b/bundle/permissions/workspace_root.go
@@ -3,6 +3,7 @@ package permissions
 import (
 	"context"
 	"fmt"
+	"strconv"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/libraries"
@@ -78,7 +79,7 @@ func setPermissions(ctx context.Context, w workspace.WorkspaceInterface, path st
 	}
 
 	_, err = w.SetPermissions(ctx, workspace.WorkspaceObjectPermissionsRequest{
-		WorkspaceObjectId:   fmt.Sprint(obj.ObjectId),
+		WorkspaceObjectId:   strconv.FormatInt(obj.ObjectId, 10),
 		WorkspaceObjectType: "directories",
 		AccessControlList:   permissions,
 	})
diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go
index 2dc9623bd..16595611f 100644
--- a/bundle/phases/deploy.go
+++ b/bundle/phases/deploy.go
@@ -2,7 +2,7 @@ package phases
 
 import (
 	"context"
-	"fmt"
+	"errors"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/artifacts"
@@ -54,7 +54,7 @@ func filterDeleteOrRecreateActions(changes []*tfjson.ResourceChange, resourceTyp
 func approvalForDeploy(ctx context.Context, b *bundle.Bundle) (bool, error) {
 	tf := b.Terraform
 	if tf == nil {
-		return false, fmt.Errorf("terraform not initialized")
+		return false, errors.New("terraform not initialized")
 	}
 
 	// read plan file
@@ -111,7 +111,7 @@ is removed from the catalog, but the underlying files are not deleted:`
 	}
 
 	if !cmdio.IsPromptSupported(ctx) {
-		return false, fmt.Errorf("the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed")
+		return false, errors.New("the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed")
 	}
 
 	cmdio.LogString(ctx, "")
diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go
index 6eb8b6a01..05a41dea2 100644
--- a/bundle/phases/destroy.go
+++ b/bundle/phases/destroy.go
@@ -3,7 +3,6 @@ package phases
 import (
 	"context"
 	"errors"
-	"fmt"
 	"net/http"
 
 	"github.com/databricks/cli/bundle"
@@ -34,7 +33,7 @@ func assertRootPathExists(ctx context.Context, b *bundle.Bundle) (bool, error) {
 func approvalForDestroy(ctx context.Context, b *bundle.Bundle) (bool, error) {
 	tf := b.Terraform
 	if tf == nil {
-		return false, fmt.Errorf("terraform not initialized")
+		return false, errors.New("terraform not initialized")
 	}
 
 	// read plan file
@@ -63,7 +62,7 @@ func approvalForDestroy(ctx context.Context, b *bundle.Bundle) (bool, error) {
 
 	}
 
-	cmdio.LogString(ctx, fmt.Sprintf("All files and directories at the following location will be deleted: %s", b.Config.Workspace.RootPath))
+	cmdio.LogString(ctx, "All files and directories at the following location will be deleted: "+b.Config.Workspace.RootPath)
 	cmdio.LogString(ctx, "")
 
 	if b.AutoApprove {
diff --git a/bundle/root.go b/bundle/root.go
index efc21e0ca..9ea9a8c13 100644
--- a/bundle/root.go
+++ b/bundle/root.go
@@ -2,6 +2,7 @@ package bundle
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 
@@ -21,7 +22,7 @@ func getRootEnv(ctx context.Context) (string, error) {
 	}
 	stat, err := os.Stat(path)
 	if err == nil && !stat.IsDir() {
-		err = fmt.Errorf("not a directory")
+		err = errors.New("not a directory")
 	}
 	if err != nil {
 		return "", fmt.Errorf(`invalid bundle root %s="%s": %w`, env.RootVariable, path, err)
diff --git a/bundle/run/job.go b/bundle/run/job.go
index b43db9184..2489ca619 100644
--- a/bundle/run/job.go
+++ b/bundle/run/job.go
@@ -3,6 +3,7 @@ package run
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strconv"
 	"time"
@@ -181,13 +182,13 @@ func (r *jobRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, e
 	// callback to log progress events. Called on every poll request
 	progressLogger, ok := cmdio.FromContext(ctx)
 	if !ok {
-		return nil, fmt.Errorf("no progress logger found")
+		return nil, errors.New("no progress logger found")
 	}
 	logProgress := logProgressCallback(ctx, progressLogger)
 
 	waiter, err := w.Jobs.RunNow(ctx, *req)
 	if err != nil {
-		return nil, fmt.Errorf("cannot start job")
+		return nil, errors.New("cannot start job")
 	}
 
 	if opts.NoWait {
@@ -266,7 +267,7 @@ func (r *jobRunner) convertPythonParams(opts *Options) error {
 
 	if len(opts.Job.pythonParams) > 0 {
 		if _, ok := opts.Job.notebookParams["__python_params"]; ok {
-			return fmt.Errorf("can't use __python_params as notebook param, the name is reserved for internal use")
+			return errors.New("can't use __python_params as notebook param, the name is reserved for internal use")
 		}
 		p, err := json.Marshal(opts.Job.pythonParams)
 		if err != nil {
diff --git a/bundle/run/job_options.go b/bundle/run/job_options.go
index 6a03dff95..7db8e72cd 100644
--- a/bundle/run/job_options.go
+++ b/bundle/run/job_options.go
@@ -1,7 +1,7 @@
 package run
 
 import (
-	"fmt"
+	"errors"
 	"strconv"
 
 	"github.com/databricks/cli/bundle/config/resources"
@@ -60,16 +60,16 @@ func (o *JobOptions) hasJobParametersConfigured() bool {
 // Validate returns if the combination of options is valid.
 func (o *JobOptions) Validate(job *resources.Job) error {
 	if job == nil {
-		return fmt.Errorf("job not defined")
+		return errors.New("job not defined")
 	}
 
 	// Ensure mutual exclusion on job parameters and task parameters.
 	hasJobParams := len(job.Parameters) > 0
 	if hasJobParams && o.hasTaskParametersConfigured() {
-		return fmt.Errorf("the job to run defines job parameters; specifying task parameters is not allowed")
+		return errors.New("the job to run defines job parameters; specifying task parameters is not allowed")
 	}
 	if !hasJobParams && o.hasJobParametersConfigured() {
-		return fmt.Errorf("the job to run does not define job parameters; specifying job parameters is not allowed")
+		return errors.New("the job to run does not define job parameters; specifying job parameters is not allowed")
 	}
 
 	return nil
@@ -80,7 +80,7 @@ func (o *JobOptions) validatePipelineParams() (*jobs.PipelineParams, error) {
 		return nil, nil
 	}
 
-	defaultErr := fmt.Errorf("job run argument --pipeline-params only supports `full_refresh=<bool>`")
+	defaultErr := errors.New("job run argument --pipeline-params only supports `full_refresh=<bool>`")
 	v, ok := o.pipelineParams["full_refresh"]
 	if !ok {
 		return nil, defaultErr
diff --git a/bundle/run/output/job.go b/bundle/run/output/job.go
index 6199ac2f7..2ac974cd5 100644
--- a/bundle/run/output/job.go
+++ b/bundle/run/output/job.go
@@ -47,7 +47,7 @@ func (out *JobOutput) String() (string, error) {
 		}
 		result.WriteString("=======\n")
 		result.WriteString(fmt.Sprintf("Task %s:\n", v.TaskKey))
-		result.WriteString(fmt.Sprintf("%s\n", taskString))
+		result.WriteString(taskString + "\n")
 	}
 	return result.String(), nil
 }
diff --git a/bundle/run/output/task.go b/bundle/run/output/task.go
index 1ef78a8c3..53b989e88 100644
--- a/bundle/run/output/task.go
+++ b/bundle/run/output/task.go
@@ -2,7 +2,6 @@ package output
 
 import (
 	"encoding/json"
-	"fmt"
 
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 )
@@ -27,7 +26,7 @@ func structToString(val any) (string, error) {
 
 func (out *NotebookOutput) String() (string, error) {
 	if out.Truncated {
-		return fmt.Sprintf("%s\n[truncated...]\n", out.Result), nil
+		return out.Result + "\n[truncated...]\n", nil
 	}
 	return out.Result, nil
 }
@@ -42,7 +41,7 @@ func (out *DbtOutput) String() (string, error) {
 	// JSON is used because it's a convenient representation.
 	// If user needs machine parsable output, they can use the --output json
 	// flag
-	return fmt.Sprintf("Dbt Task Output:\n%s", outputString), nil
+	return "Dbt Task Output:\n" + outputString, nil
 }
 
 func (out *SqlOutput) String() (string, error) {
@@ -55,12 +54,12 @@ func (out *SqlOutput) String() (string, error) {
 	// JSON is used because it's a convenient representation.
 	// If user needs machine parsable output, they can use the --output json
 	// flag
-	return fmt.Sprintf("SQL Task Output:\n%s", outputString), nil
+	return "SQL Task Output:\n" + outputString, nil
 }
 
 func (out *LogsOutput) String() (string, error) {
 	if out.LogsTruncated {
-		return fmt.Sprintf("%s\n[truncated...]\n", out.Logs), nil
+		return out.Logs + "\n[truncated...]\n", nil
 	}
 	return out.Logs, nil
 }
diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go
index c447f044a..bdcf0f142 100644
--- a/bundle/run/pipeline.go
+++ b/bundle/run/pipeline.go
@@ -2,6 +2,7 @@ package run
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"time"
 
@@ -33,7 +34,7 @@ func (r *pipelineRunner) logEvent(ctx context.Context, event pipelines.PipelineE
 	if event.Error != nil && len(event.Error.Exceptions) > 0 {
 		logString += "trace for most recent exception: \n"
 		for i := range len(event.Error.Exceptions) {
-			logString += fmt.Sprintf("%s\n", event.Error.Exceptions[i].Message)
+			logString += event.Error.Exceptions[i].Message + "\n"
 		}
 	}
 	if logString != "" {
@@ -107,7 +108,7 @@ func (r *pipelineRunner) Run(ctx context.Context, opts *Options) (output.RunOutp
 	updateTracker := progress.NewUpdateTracker(pipelineID, updateID, w)
 	progressLogger, ok := cmdio.FromContext(ctx)
 	if !ok {
-		return nil, fmt.Errorf("no progress logger found")
+		return nil, errors.New("no progress logger found")
 	}
 
 	// Log the pipeline update URL as soon as it is available.
@@ -144,7 +145,7 @@ func (r *pipelineRunner) Run(ctx context.Context, opts *Options) (output.RunOutp
 
 		if state == pipelines.UpdateInfoStateCanceled {
 			log.Infof(ctx, "Update was cancelled!")
-			return nil, fmt.Errorf("update cancelled")
+			return nil, errors.New("update cancelled")
 		}
 		if state == pipelines.UpdateInfoStateFailed {
 			log.Infof(ctx, "Update has failed!")
@@ -152,7 +153,7 @@ func (r *pipelineRunner) Run(ctx context.Context, opts *Options) (output.RunOutp
 			if err != nil {
 				return nil, err
 			}
-			return nil, fmt.Errorf("update failed")
+			return nil, errors.New("update failed")
 		}
 		if state == pipelines.UpdateInfoStateCompleted {
 			log.Infof(ctx, "Update has completed successfully!")
diff --git a/bundle/run/progress/pipeline.go b/bundle/run/progress/pipeline.go
index b82dd7abd..ce92c4cde 100644
--- a/bundle/run/progress/pipeline.go
+++ b/bundle/run/progress/pipeline.go
@@ -33,7 +33,7 @@ func (event *ProgressEvent) String() string {
 	// construct error string if level=`Error`
 	if event.Level == pipelines.EventLevelError && event.Error != nil {
 		for _, exception := range event.Error.Exceptions {
-			result.WriteString(fmt.Sprintf("\n%s", exception.Message))
+			result.WriteString("\n" + exception.Message)
 		}
 	}
 	return result.String()
diff --git a/bundle/tests/run_as_test.go b/bundle/tests/run_as_test.go
index 113a6140b..03ff51ec5 100644
--- a/bundle/tests/run_as_test.go
+++ b/bundle/tests/run_as_test.go
@@ -2,7 +2,6 @@ package config_tests
 
 import (
 	"context"
-	"fmt"
 	"testing"
 
 	"github.com/databricks/cli/bundle"
@@ -219,7 +218,7 @@ func TestRunAsErrorNeitherUserOrSpSpecified(t *testing.T) {
 
 	for _, tc := range tcases {
 		t.Run(tc.name, func(t *testing.T) {
-			bundlePath := fmt.Sprintf("./run_as/not_allowed/neither_sp_nor_user/%s", tc.name)
+			bundlePath := "./run_as/not_allowed/neither_sp_nor_user/" + tc.name
 			b := load(t, bundlePath)
 
 			ctx := context.Background()
diff --git a/bundle/trampoline/python_wheel.go b/bundle/trampoline/python_wheel.go
index 8e309a625..075804479 100644
--- a/bundle/trampoline/python_wheel.go
+++ b/bundle/trampoline/python_wheel.go
@@ -2,6 +2,7 @@ package trampoline
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strconv"
 	"strings"
@@ -147,7 +148,7 @@ func (t *pythonTrampoline) GetTemplateData(task *jobs.Task) (map[string]any, err
 
 func (t *pythonTrampoline) generateParameters(task *jobs.PythonWheelTask) (string, error) {
 	if task.Parameters != nil && task.NamedParameters != nil {
-		return "", fmt.Errorf("not allowed to pass both paramaters and named_parameters")
+		return "", errors.New("not allowed to pass both paramaters and named_parameters")
 	}
 	params := append([]string{task.PackageName}, task.Parameters...)
 	for k, v := range task.NamedParameters {
diff --git a/bundle/trampoline/trampoline_test.go b/bundle/trampoline/trampoline_test.go
index 3c5d18570..6e6b8db48 100644
--- a/bundle/trampoline/trampoline_test.go
+++ b/bundle/trampoline/trampoline_test.go
@@ -2,7 +2,7 @@ package trampoline
 
 import (
 	"context"
-	"fmt"
+	"errors"
 	"os"
 	"path/filepath"
 	"testing"
@@ -30,7 +30,7 @@ func (f *functions) GetTasks(b *bundle.Bundle) []TaskWithJobKey {
 
 func (f *functions) GetTemplateData(task *jobs.Task) (map[string]any, error) {
 	if task.PythonWheelTask == nil {
-		return nil, fmt.Errorf("PythonWheelTask cannot be nil")
+		return nil, errors.New("PythonWheelTask cannot be nil")
 	}
 
 	data := make(map[string]any)
diff --git a/cmd/auth/auth.go b/cmd/auth/auth.go
index ceceae25c..4261e93e7 100644
--- a/cmd/auth/auth.go
+++ b/cmd/auth/auth.go
@@ -2,7 +2,7 @@ package auth
 
 import (
 	"context"
-	"fmt"
+	"errors"
 
 	"github.com/databricks/cli/libs/auth"
 	"github.com/databricks/cli/libs/cmdio"
@@ -36,7 +36,7 @@ GCP: https://docs.gcp.databricks.com/dev-tools/auth/index.html`,
 
 func promptForHost(ctx context.Context) (string, error) {
 	if !cmdio.IsInTTY(ctx) {
-		return "", fmt.Errorf("the command is being run in a non-interactive environment, please specify a host using --host")
+		return "", errors.New("the command is being run in a non-interactive environment, please specify a host using --host")
 	}
 
 	prompt := cmdio.Prompt(ctx)
@@ -46,7 +46,7 @@ func promptForHost(ctx context.Context) (string, error) {
 
 func promptForAccountID(ctx context.Context) (string, error) {
 	if !cmdio.IsInTTY(ctx) {
-		return "", fmt.Errorf("the command is being run in a non-interactive environment, please specify an account ID using --account-id")
+		return "", errors.New("the command is being run in a non-interactive environment, please specify an account ID using --account-id")
 	}
 
 	prompt := cmdio.Prompt(ctx)
diff --git a/cmd/auth/describe_test.go b/cmd/auth/describe_test.go
index 7f5f900d4..35e0c6e64 100644
--- a/cmd/auth/describe_test.go
+++ b/cmd/auth/describe_test.go
@@ -2,7 +2,7 @@ package auth
 
 import (
 	"context"
-	"fmt"
+	"errors"
 	"testing"
 
 	"github.com/databricks/cli/cmd/root"
@@ -102,7 +102,7 @@ func TestGetWorkspaceAuthStatusError(t *testing.T) {
 			"token":     "test-token",
 			"auth_type": "azure-cli",
 		})
-		return cfg, false, fmt.Errorf("auth error")
+		return cfg, false, errors.New("auth error")
 	})
 	require.NoError(t, err)
 	require.NotNil(t, status)
@@ -151,7 +151,7 @@ func TestGetWorkspaceAuthStatusSensitive(t *testing.T) {
 			"token":     "test-token",
 			"auth_type": "azure-cli",
 		})
-		return cfg, false, fmt.Errorf("auth error")
+		return cfg, false, errors.New("auth error")
 	})
 	require.NoError(t, err)
 	require.NotNil(t, status)
diff --git a/cmd/auth/env.go b/cmd/auth/env.go
index 52b7cbbfd..11149af8c 100644
--- a/cmd/auth/env.go
+++ b/cmd/auth/env.go
@@ -23,9 +23,9 @@ func canonicalHost(host string) (string, error) {
 	}
 	// If the host is empty, assume the scheme wasn't included.
 	if parsedHost.Host == "" {
-		return fmt.Sprintf("https://%s", host), nil
+		return "https://" + host, nil
 	}
-	return fmt.Sprintf("https://%s", parsedHost.Host), nil
+	return "https://" + parsedHost.Host, nil
 }
 
 var ErrNoMatchingProfiles = errors.New("no matching profiles found")
diff --git a/cmd/auth/login.go b/cmd/auth/login.go
index c98676599..a6d0bf4cc 100644
--- a/cmd/auth/login.go
+++ b/cmd/auth/login.go
@@ -176,7 +176,7 @@ depends on the existing profiles you have set in your configuration file
 func setHostAndAccountId(ctx context.Context, profileName string, persistentAuth *auth.PersistentAuth, args []string) error {
 	// If both [HOST] and --host are provided, return an error.
 	if len(args) > 0 && persistentAuth.Host != "" {
-		return fmt.Errorf("please only provide a host as an argument or a flag, not both")
+		return errors.New("please only provide a host as an argument or a flag, not both")
 	}
 
 	profiler := profile.GetProfiler(ctx)
diff --git a/cmd/bundle/destroy.go b/cmd/bundle/destroy.go
index 711abbcd7..0b2f14875 100644
--- a/cmd/bundle/destroy.go
+++ b/cmd/bundle/destroy.go
@@ -2,7 +2,7 @@ package bundle
 
 import (
 	"context"
-	"fmt"
+	"errors"
 	"os"
 
 	"github.com/databricks/cli/bundle"
@@ -49,16 +49,16 @@ func newDestroyCommand() *cobra.Command {
 		// we require auto-approve for non tty terminals since interactive consent
 		// is not possible
 		if !term.IsTerminal(int(os.Stderr.Fd())) && !autoApprove {
-			return fmt.Errorf("please specify --auto-approve to skip interactive confirmation checks for non tty consoles")
+			return errors.New("please specify --auto-approve to skip interactive confirmation checks for non tty consoles")
 		}
 
 		// Check auto-approve is selected for json logging
 		logger, ok := cmdio.FromContext(ctx)
 		if !ok {
-			return fmt.Errorf("progress logger not found")
+			return errors.New("progress logger not found")
 		}
 		if logger.Mode == flags.ModeJson && !autoApprove {
-			return fmt.Errorf("please specify --auto-approve since selected logging format is json")
+			return errors.New("please specify --auto-approve since selected logging format is json")
 		}
 
 		diags = bundle.Apply(ctx, b, bundle.Seq(
diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go
index f196bbe62..fa3c91b2a 100644
--- a/cmd/bundle/generate/dashboard.go
+++ b/cmd/bundle/generate/dashboard.go
@@ -96,7 +96,7 @@ func (d *dashboard) resolveFromPath(ctx context.Context, b *bundle.Bundle) (stri
 		return "", diag.Diagnostics{
 			{
 				Severity: diag.Error,
-				Summary:  fmt.Sprintf("expected a dashboard, found a %s", found),
+				Summary:  "expected a dashboard, found a " + found,
 			},
 		}
 	}
@@ -188,7 +188,7 @@ func (d *dashboard) saveSerializedDashboard(_ context.Context, b *bundle.Bundle,
 
 func (d *dashboard) saveConfiguration(ctx context.Context, b *bundle.Bundle, dashboard *dashboards.Dashboard, key string) error {
 	// Save serialized dashboard definition to the dashboard directory.
-	dashboardBasename := fmt.Sprintf("%s.lvdash.json", key)
+	dashboardBasename := key + ".lvdash.json"
 	dashboardPath := filepath.Join(d.dashboardDir, dashboardBasename)
 	err := d.saveSerializedDashboard(ctx, b, dashboard, dashboardPath)
 	if err != nil {
@@ -215,7 +215,7 @@ func (d *dashboard) saveConfiguration(ctx context.Context, b *bundle.Bundle, das
 	}
 
 	// Save the configuration to the resource directory.
-	resourcePath := filepath.Join(d.resourceDir, fmt.Sprintf("%s.dashboard.yml", key))
+	resourcePath := filepath.Join(d.resourceDir, key+".dashboard.yml")
 	saver := yamlsaver.NewSaverWithStyle(map[string]yaml.Style{
 		"display_name": yaml.DoubleQuotedStyle,
 	})
diff --git a/cmd/bundle/generate/job.go b/cmd/bundle/generate/job.go
index 9ac41e3cb..827d270e5 100644
--- a/cmd/bundle/generate/job.go
+++ b/cmd/bundle/generate/job.go
@@ -85,8 +85,8 @@ func NewGenerateJobCommand() *cobra.Command {
 			return err
 		}
 
-		oldFilename := filepath.Join(configDir, fmt.Sprintf("%s.yml", jobKey))
-		filename := filepath.Join(configDir, fmt.Sprintf("%s.job.yml", jobKey))
+		oldFilename := filepath.Join(configDir, jobKey+".yml")
+		filename := filepath.Join(configDir, jobKey+".job.yml")
 
 		// User might continuously run generate command to update their bundle jobs with any changes made in Databricks UI.
 		// Due to changing in the generated file names, we need to first rename existing resource file to the new name.
@@ -107,7 +107,7 @@ func NewGenerateJobCommand() *cobra.Command {
 			return err
 		}
 
-		cmdio.LogString(ctx, fmt.Sprintf("Job configuration successfully saved to %s", filename))
+		cmdio.LogString(ctx, "Job configuration successfully saved to "+filename)
 		return nil
 	}
 
diff --git a/cmd/bundle/generate/pipeline.go b/cmd/bundle/generate/pipeline.go
index 910baa45f..863b0b2f7 100644
--- a/cmd/bundle/generate/pipeline.go
+++ b/cmd/bundle/generate/pipeline.go
@@ -85,8 +85,8 @@ func NewGeneratePipelineCommand() *cobra.Command {
 			return err
 		}
 
-		oldFilename := filepath.Join(configDir, fmt.Sprintf("%s.yml", pipelineKey))
-		filename := filepath.Join(configDir, fmt.Sprintf("%s.pipeline.yml", pipelineKey))
+		oldFilename := filepath.Join(configDir, pipelineKey+".yml")
+		filename := filepath.Join(configDir, pipelineKey+".pipeline.yml")
 
 		// User might continuously run generate command to update their bundle jobs with any changes made in Databricks UI.
 		// Due to changing in the generated file names, we need to first rename existing resource file to the new name.
@@ -109,7 +109,7 @@ func NewGeneratePipelineCommand() *cobra.Command {
 			return err
 		}
 
-		cmdio.LogString(ctx, fmt.Sprintf("Pipeline configuration successfully saved to %s", filename))
+		cmdio.LogString(ctx, "Pipeline configuration successfully saved to "+filename)
 		return nil
 	}
 
diff --git a/cmd/bundle/generate/utils.go b/cmd/bundle/generate/utils.go
index 8e3764e35..dbfad9438 100644
--- a/cmd/bundle/generate/utils.go
+++ b/cmd/bundle/generate/utils.go
@@ -126,7 +126,7 @@ func (n *downloader) FlushToDisk(ctx context.Context, force bool) error {
 				return err
 			}
 
-			cmdio.LogString(errCtx, fmt.Sprintf("File successfully saved to %s", targetPath))
+			cmdio.LogString(errCtx, "File successfully saved to "+targetPath)
 			return reader.Close()
 		})
 	}
diff --git a/cmd/bundle/launch.go b/cmd/bundle/launch.go
index 0d2b4233b..3fea839c9 100644
--- a/cmd/bundle/launch.go
+++ b/cmd/bundle/launch.go
@@ -1,7 +1,7 @@
 package bundle
 
 import (
-	"fmt"
+	"errors"
 
 	"github.com/databricks/cli/cmd/root"
 	"github.com/spf13/cobra"
@@ -19,7 +19,7 @@ func newLaunchCommand() *cobra.Command {
 	}
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {
-		return fmt.Errorf("TODO")
+		return errors.New("TODO")
 		// contents, err := os.ReadFile(args[0])
 		// if err != nil {
 		// 	return err
diff --git a/cmd/bundle/open.go b/cmd/bundle/open.go
index a2ad32fd8..5a26e1ea7 100644
--- a/cmd/bundle/open.go
+++ b/cmd/bundle/open.go
@@ -44,7 +44,7 @@ func resolveOpenArgument(ctx context.Context, b *bundle.Bundle, args []string) (
 	}
 
 	if len(args) < 1 {
-		return "", fmt.Errorf("expected a KEY of the resource to open")
+		return "", errors.New("expected a KEY of the resource to open")
 	}
 
 	return args[0], nil
@@ -113,7 +113,7 @@ func newOpenCommand() *cobra.Command {
 		// Confirm that the resource has a URL.
 		url := ref.Resource.GetURL()
 		if url == "" {
-			return fmt.Errorf("resource does not have a URL associated with it (has it been deployed?)")
+			return errors.New("resource does not have a URL associated with it (has it been deployed?)")
 		}
 
 		return browser.OpenURL(url)
diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go
index 3bcebddd5..df35d7222 100644
--- a/cmd/bundle/run.go
+++ b/cmd/bundle/run.go
@@ -3,6 +3,7 @@ package bundle
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 
 	"github.com/databricks/cli/bundle"
@@ -48,7 +49,7 @@ func resolveRunArgument(ctx context.Context, b *bundle.Bundle, args []string) (s
 	}
 
 	if len(args) < 1 {
-		return "", nil, fmt.Errorf("expected a KEY of the resource to run")
+		return "", nil, errors.New("expected a KEY of the resource to run")
 	}
 
 	return args[0], args[1:], nil
diff --git a/cmd/bundle/test.go b/cmd/bundle/test.go
index 4d30e727d..794575220 100644
--- a/cmd/bundle/test.go
+++ b/cmd/bundle/test.go
@@ -1,7 +1,7 @@
 package bundle
 
 import (
-	"fmt"
+	"errors"
 
 	"github.com/spf13/cobra"
 )
@@ -17,7 +17,7 @@ func newTestCommand() *cobra.Command {
 	}
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {
-		return fmt.Errorf("TODO")
+		return errors.New("TODO")
 		// results := project.RunPythonOnDev(cmd.Context(), `return 1`)
 		// if results.Failed() {
 		// 	return results.Err()
diff --git a/cmd/bundle/validate.go b/cmd/bundle/validate.go
index 3b50cc258..daeb7426d 100644
--- a/cmd/bundle/validate.go
+++ b/cmd/bundle/validate.go
@@ -2,6 +2,7 @@ package bundle
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 
 	"github.com/databricks/cli/bundle"
@@ -39,7 +40,7 @@ func newValidateCommand() *cobra.Command {
 			if err := diags.Error(); err != nil {
 				return diags.Error()
 			} else {
-				return fmt.Errorf("invariant failed: returned bundle is nil")
+				return errors.New("invariant failed: returned bundle is nil")
 			}
 		}
 
diff --git a/cmd/configure/configure.go b/cmd/configure/configure.go
index 895a5902c..4a6568d06 100644
--- a/cmd/configure/configure.go
+++ b/cmd/configure/configure.go
@@ -1,6 +1,7 @@
 package configure
 
 import (
+	"errors"
 	"fmt"
 
 	"github.com/databricks/cli/libs/cmdio"
@@ -62,12 +63,12 @@ func configureInteractive(cmd *cobra.Command, flags *configureFlags, cfg *config
 
 func configureNonInteractive(cmd *cobra.Command, flags *configureFlags, cfg *config.Config) error {
 	if cfg.Host == "" {
-		return fmt.Errorf("host must be set in non-interactive mode")
+		return errors.New("host must be set in non-interactive mode")
 	}
 
 	// Check presence of cluster ID before reading token to fail fast.
 	if flags.ConfigureCluster && cfg.ClusterID == "" {
-		return fmt.Errorf("cluster ID must be set in non-interactive mode")
+		return errors.New("cluster ID must be set in non-interactive mode")
 	}
 
 	// Read token from stdin if not already set.
diff --git a/cmd/configure/host.go b/cmd/configure/host.go
index 781c12387..0a454c6d1 100644
--- a/cmd/configure/host.go
+++ b/cmd/configure/host.go
@@ -1,7 +1,7 @@
 package configure
 
 import (
-	"fmt"
+	"errors"
 	"net/url"
 )
 
@@ -11,10 +11,10 @@ func validateHost(s string) error {
 		return err
 	}
 	if u.Host == "" || u.Scheme != "https" {
-		return fmt.Errorf("must start with https://")
+		return errors.New("must start with https://")
 	}
 	if u.Path != "" && u.Path != "/" {
-		return fmt.Errorf("must use empty path")
+		return errors.New("must use empty path")
 	}
 	return nil
 }
diff --git a/cmd/labs/github/repositories.go b/cmd/labs/github/repositories.go
index 850cdb1cb..afdf7aeb2 100644
--- a/cmd/labs/github/repositories.go
+++ b/cmd/labs/github/repositories.go
@@ -12,7 +12,7 @@ import (
 const repositoryCacheTTL = 24 * time.Hour
 
 func NewRepositoryCache(org, cacheDir string) *repositoryCache {
-	filename := fmt.Sprintf("%s-repositories", org)
+	filename := org + "-repositories"
 	return &repositoryCache{
 		cache: localcache.NewLocalCache[Repositories](cacheDir, filename, repositoryCacheTTL),
 		Org:   org,
diff --git a/cmd/labs/installed.go b/cmd/labs/installed.go
index e4249c9ff..9982cc1f0 100644
--- a/cmd/labs/installed.go
+++ b/cmd/labs/installed.go
@@ -1,6 +1,7 @@
 package labs
 
 import (
+	"errors"
 	"fmt"
 
 	"github.com/databricks/cli/cmd/labs/project"
@@ -49,7 +50,7 @@ func newInstalledCommand() *cobra.Command {
 				})
 			}
 			if len(info.Projects) == 0 {
-				return fmt.Errorf("no projects installed")
+				return errors.New("no projects installed")
 			}
 			return cmdio.Render(ctx, info)
 		},
diff --git a/cmd/labs/localcache/jsonfile.go b/cmd/labs/localcache/jsonfile.go
index 6540e4ac2..50ed372f5 100644
--- a/cmd/labs/localcache/jsonfile.go
+++ b/cmd/labs/localcache/jsonfile.go
@@ -93,7 +93,7 @@ func (r *LocalCache[T]) writeCache(ctx context.Context, data T) (T, error) {
 }
 
 func (r *LocalCache[T]) FileName() string {
-	return filepath.Join(r.dir, fmt.Sprintf("%s.json", r.name))
+	return filepath.Join(r.dir, r.name+".json")
 }
 
 func (r *LocalCache[T]) loadCache() (*cached[T], error) {
diff --git a/cmd/labs/localcache/jsonfile_test.go b/cmd/labs/localcache/jsonfile_test.go
index 9d42c6179..8172b7d14 100644
--- a/cmd/labs/localcache/jsonfile_test.go
+++ b/cmd/labs/localcache/jsonfile_test.go
@@ -3,7 +3,6 @@ package localcache
 import (
 	"context"
 	"errors"
-	"fmt"
 	"net/url"
 	"runtime"
 	"testing"
@@ -115,7 +114,7 @@ func TestFolderDisappears(t *testing.T) {
 func TestRefreshFails(t *testing.T) {
 	c := NewLocalCache[int64](t.TempDir(), "time", 1*time.Minute)
 	tick := func() (int64, error) {
-		return 0, fmt.Errorf("nope")
+		return 0, errors.New("nope")
 	}
 	ctx := context.Background()
 	_, err := c.Load(ctx, tick)
diff --git a/cmd/labs/project/installer.go b/cmd/labs/project/installer.go
index 041415964..7d31623bb 100644
--- a/cmd/labs/project/installer.go
+++ b/cmd/labs/project/installer.go
@@ -175,7 +175,7 @@ func (i *installer) login(ctx context.Context) (*databricks.WorkspaceClient, err
 		return nil, fmt.Errorf("valid: %w", err)
 	}
 	if !i.HasAccountLevelCommands() && cfg.IsAccountClient() {
-		return nil, fmt.Errorf("got account-level client, but no account-level commands")
+		return nil, errors.New("got account-level client, but no account-level commands")
 	}
 	lc := &loginConfig{Entrypoint: i.Installer.Entrypoint}
 	w, err := lc.askWorkspace(ctx, cfg)
@@ -200,10 +200,10 @@ func (i *installer) downloadLibrary(ctx context.Context) error {
 	libTarget := i.LibDir()
 	// we may support wheels, jars, and golang binaries. but those are not zipballs
 	if i.IsZipball() {
-		feedback <- fmt.Sprintf("Downloading and unpacking zipball for %s", i.version)
+		feedback <- "Downloading and unpacking zipball for " + i.version
 		return i.downloadAndUnpackZipball(ctx, libTarget)
 	}
-	return fmt.Errorf("we only support zipballs for now")
+	return errors.New("we only support zipballs for now")
 }
 
 func (i *installer) downloadAndUnpackZipball(ctx context.Context, libTarget string) error {
@@ -234,7 +234,7 @@ func (i *installer) setupPythonVirtualEnvironment(ctx context.Context, w *databr
 	log.Debugf(ctx, "Detected Python %s at: %s", py.Version, py.Path)
 	venvPath := i.virtualEnvPath(ctx)
 	log.Debugf(ctx, "Creating Python Virtual Environment at: %s", venvPath)
-	feedback <- fmt.Sprintf("Creating Virtual Environment with Python %s", py.Version)
+	feedback <- "Creating Virtual Environment with Python " + py.Version
 	_, err = process.Background(ctx, []string{py.Path, "-m", "venv", venvPath})
 	if err != nil {
 		return fmt.Errorf("create venv: %w", err)
@@ -251,8 +251,8 @@ func (i *installer) setupPythonVirtualEnvironment(ctx context.Context, w *databr
 		if !ok {
 			return fmt.Errorf("unsupported runtime: %s", cluster.SparkVersion)
 		}
-		feedback <- fmt.Sprintf("Installing Databricks Connect v%s", runtimeVersion)
-		pipSpec := fmt.Sprintf("databricks-connect==%s", runtimeVersion)
+		feedback <- "Installing Databricks Connect v" + runtimeVersion
+		pipSpec := "databricks-connect==" + runtimeVersion
 		err = i.installPythonDependencies(ctx, pipSpec)
 		if err != nil {
 			return fmt.Errorf("dbconnect: %w", err)
diff --git a/cmd/labs/show.go b/cmd/labs/show.go
index c36f0bda3..e8c876d8b 100644
--- a/cmd/labs/show.go
+++ b/cmd/labs/show.go
@@ -1,7 +1,7 @@
 package labs
 
 import (
-	"fmt"
+	"errors"
 
 	"github.com/databricks/cli/cmd/labs/project"
 	"github.com/databricks/cli/cmd/root"
@@ -34,7 +34,7 @@ func newShowCommand() *cobra.Command {
 				return err
 			}
 			if len(installed) == 0 {
-				return fmt.Errorf("no projects found")
+				return errors.New("no projects found")
 			}
 			name := args[0]
 			for _, v := range installed {
diff --git a/cmd/root/auth.go b/cmd/root/auth.go
index 07ab48399..49abfd414 100644
--- a/cmd/root/auth.go
+++ b/cmd/root/auth.go
@@ -26,7 +26,7 @@ type ErrNoWorkspaceProfiles struct {
 }
 
 func (e ErrNoWorkspaceProfiles) Error() string {
-	return fmt.Sprintf("%s does not contain workspace profiles; please create one by running 'databricks configure'", e.path)
+	return e.path + " does not contain workspace profiles; please create one by running 'databricks configure'"
 }
 
 type ErrNoAccountProfiles struct {
@@ -34,7 +34,7 @@ type ErrNoAccountProfiles struct {
 }
 
 func (e ErrNoAccountProfiles) Error() string {
-	return fmt.Sprintf("%s does not contain account profiles", e.path)
+	return e.path + " does not contain account profiles"
 }
 
 func initProfileFlag(cmd *cobra.Command) {
@@ -253,7 +253,7 @@ func AskForWorkspaceProfile(ctx context.Context) (string, error) {
 		return profiles[0].Name, nil
 	}
 	i, _, err := cmdio.RunSelect(ctx, &promptui.Select{
-		Label:             fmt.Sprintf("Workspace profiles defined in %s", path),
+		Label:             "Workspace profiles defined in " + path,
 		Items:             profiles,
 		Searcher:          profiles.SearchCaseInsensitive,
 		StartInSearchMode: true,
@@ -287,7 +287,7 @@ func AskForAccountProfile(ctx context.Context) (string, error) {
 		return profiles[0].Name, nil
 	}
 	i, _, err := cmdio.RunSelect(ctx, &promptui.Select{
-		Label:             fmt.Sprintf("Account profiles defined in %s", path),
+		Label:             "Account profiles defined in " + path,
 		Items:             profiles,
 		Searcher:          profiles.SearchCaseInsensitive,
 		StartInSearchMode: true,
diff --git a/cmd/root/progress_logger.go b/cmd/root/progress_logger.go
index 1458de13a..0cc49b2ac 100644
--- a/cmd/root/progress_logger.go
+++ b/cmd/root/progress_logger.go
@@ -2,7 +2,7 @@ package root
 
 import (
 	"context"
-	"fmt"
+	"errors"
 	"os"
 
 	"github.com/databricks/cli/libs/cmdio"
@@ -37,7 +37,7 @@ func (f *progressLoggerFlag) initializeContext(ctx context.Context) (context.Con
 
 	if f.log.level.String() != "disabled" && f.log.file.String() == "stderr" &&
 		f.ProgressLogFormat == flags.ModeInplace {
-		return nil, fmt.Errorf("inplace progress logging cannot be used when log-file is stderr")
+		return nil, errors.New("inplace progress logging cannot be used when log-file is stderr")
 	}
 
 	format := f.ProgressLogFormat
diff --git a/cmd/sync/completion.go b/cmd/sync/completion.go
index 422147713..5a65dd528 100644
--- a/cmd/sync/completion.go
+++ b/cmd/sync/completion.go
@@ -2,7 +2,6 @@ package sync
 
 import (
 	"context"
-	"fmt"
 	"path"
 	"strings"
 
@@ -52,8 +51,8 @@ func completeRemotePath(
 	}
 
 	prefixes := []string{
-		path.Clean(fmt.Sprintf("/Users/%s", me.UserName)) + "/",
-		path.Clean(fmt.Sprintf("/Repos/%s", me.UserName)) + "/",
+		path.Clean("/Users/"+me.UserName) + "/",
+		path.Clean("/Repos/"+me.UserName) + "/",
 	}
 
 	validPrefix := false
diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go
index cd2167a19..dea40f96a 100644
--- a/cmd/sync/sync.go
+++ b/cmd/sync/sync.go
@@ -2,6 +2,7 @@ package sync
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -29,7 +30,7 @@ type syncFlags struct {
 
 func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, args []string, b *bundle.Bundle) (*sync.SyncOptions, error) {
 	if len(args) > 0 {
-		return nil, fmt.Errorf("SRC and DST are not configurable in the context of a bundle")
+		return nil, errors.New("SRC and DST are not configurable in the context of a bundle")
 	}
 
 	opts, err := files.GetSyncOptions(cmd.Context(), bundle.ReadOnly(b))
diff --git a/cmd/workspace/repos/overrides.go b/cmd/workspace/repos/overrides.go
index aad38ecc7..561921623 100644
--- a/cmd/workspace/repos/overrides.go
+++ b/cmd/workspace/repos/overrides.go
@@ -2,6 +2,7 @@ package repos
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strconv"
 
@@ -153,7 +154,7 @@ func repoArgumentToRepoID(ctx context.Context, w *databricks.WorkspaceClient, ar
 		args = append(args, id)
 	}
 	if len(args) != 1 {
-		return 0, fmt.Errorf("expected to have the id for the corresponding repo to access")
+		return 0, errors.New("expected to have the id for the corresponding repo to access")
 	}
 	// ---- End copy from cmd/workspace/repos/repos.go ----
 
diff --git a/cmd/workspace/secrets/put_secret.go b/cmd/workspace/secrets/put_secret.go
index f24814f05..b446524f7 100644
--- a/cmd/workspace/secrets/put_secret.go
+++ b/cmd/workspace/secrets/put_secret.go
@@ -2,7 +2,7 @@ package secrets
 
 import (
 	"encoding/base64"
-	"fmt"
+	"errors"
 	"io"
 	"os"
 
@@ -67,7 +67,7 @@ func newPutSecret() *cobra.Command {
 		bytesValueChanged := cmd.Flags().Changed("bytes-value")
 		stringValueChanged := cmd.Flags().Changed("string-value")
 		if bytesValueChanged && stringValueChanged {
-			return fmt.Errorf("cannot specify both --bytes-value and --string-value")
+			return errors.New("cannot specify both --bytes-value and --string-value")
 		}
 
 		if cmd.Flags().Changed("json") {
diff --git a/cmd/workspace/workspace/overrides.go b/cmd/workspace/workspace/overrides.go
index 216e9b5d8..53438a764 100644
--- a/cmd/workspace/workspace/overrides.go
+++ b/cmd/workspace/workspace/overrides.go
@@ -36,7 +36,7 @@ func exportOverride(exportCmd *cobra.Command, exportReq *workspace.ExportRequest
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
 		if len(args) != 1 {
-			return fmt.Errorf("expected to have the absolute path of the object or directory")
+			return errors.New("expected to have the absolute path of the object or directory")
 		}
 		exportReq.Path = args[0]
 
diff --git a/integration/bundle/bind_resource_test.go b/integration/bundle/bind_resource_test.go
index 508aa3410..ba10190aa 100644
--- a/integration/bundle/bind_resource_test.go
+++ b/integration/bundle/bind_resource_test.go
@@ -1,9 +1,9 @@
 package bundle_test
 
 import (
-	"fmt"
 	"os"
 	"path/filepath"
+	"strconv"
 	"testing"
 
 	"github.com/databricks/cli/integration/internal/acc"
@@ -35,7 +35,7 @@ func TestBindJobToExistingJob(t *testing.T) {
 	})
 
 	ctx = env.Set(ctx, "BUNDLE_ROOT", bundleRoot)
-	c := testcli.NewRunner(t, ctx, "bundle", "deployment", "bind", "foo", fmt.Sprint(jobId), "--auto-approve")
+	c := testcli.NewRunner(t, ctx, "bundle", "deployment", "bind", "foo", strconv.FormatInt(jobId, 10), "--auto-approve")
 	_, _, err := c.Run()
 	require.NoError(t, err)
 
@@ -53,7 +53,7 @@ func TestBindJobToExistingJob(t *testing.T) {
 		JobId: jobId,
 	})
 	require.NoError(t, err)
-	require.Equal(t, job.Settings.Name, fmt.Sprintf("test-job-basic-%s", uniqueId))
+	require.Equal(t, job.Settings.Name, "test-job-basic-"+uniqueId)
 	require.Contains(t, job.Settings.Tasks[0].SparkPythonTask.PythonFile, "hello_world.py")
 
 	c = testcli.NewRunner(t, ctx, "bundle", "deployment", "unbind", "foo")
@@ -71,7 +71,7 @@ func TestBindJobToExistingJob(t *testing.T) {
 		JobId: jobId,
 	})
 	require.NoError(t, err)
-	require.Equal(t, job.Settings.Name, fmt.Sprintf("test-job-basic-%s", uniqueId))
+	require.Equal(t, job.Settings.Name, "test-job-basic-"+uniqueId)
 	require.Contains(t, job.Settings.Tasks[0].SparkPythonTask.PythonFile, "hello_world.py")
 }
 
@@ -96,7 +96,7 @@ func TestAbortBind(t *testing.T) {
 	// Bind should fail because prompting is not possible.
 	ctx = env.Set(ctx, "BUNDLE_ROOT", bundleRoot)
 	ctx = env.Set(ctx, "TERM", "dumb")
-	c := testcli.NewRunner(t, ctx, "bundle", "deployment", "bind", "foo", fmt.Sprint(jobId))
+	c := testcli.NewRunner(t, ctx, "bundle", "deployment", "bind", "foo", strconv.FormatInt(jobId, 10))
 
 	// Expect error suggesting to use --auto-approve
 	_, _, err := c.Run()
@@ -114,7 +114,7 @@ func TestAbortBind(t *testing.T) {
 	})
 	require.NoError(t, err)
 
-	require.NotEqual(t, job.Settings.Name, fmt.Sprintf("test-job-basic-%s", uniqueId))
+	require.NotEqual(t, job.Settings.Name, "test-job-basic-"+uniqueId)
 	require.Contains(t, job.Settings.Tasks[0].NotebookTask.NotebookPath, "test")
 }
 
@@ -143,7 +143,7 @@ func TestGenerateAndBind(t *testing.T) {
 	ctx = env.Set(ctx, "BUNDLE_ROOT", bundleRoot)
 	c := testcli.NewRunner(t, ctx, "bundle", "generate", "job",
 		"--key", "test_job_key",
-		"--existing-job-id", fmt.Sprint(jobId),
+		"--existing-job-id", strconv.FormatInt(jobId, 10),
 		"--config-dir", filepath.Join(bundleRoot, "resources"),
 		"--source-dir", filepath.Join(bundleRoot, "src"))
 	_, _, err = c.Run()
@@ -157,7 +157,7 @@ func TestGenerateAndBind(t *testing.T) {
 
 	require.Len(t, matches, 1)
 
-	c = testcli.NewRunner(t, ctx, "bundle", "deployment", "bind", "test_job_key", fmt.Sprint(jobId), "--auto-approve")
+	c = testcli.NewRunner(t, ctx, "bundle", "deployment", "bind", "test_job_key", strconv.FormatInt(jobId, 10), "--auto-approve")
 	_, _, err = c.Run()
 	require.NoError(t, err)
 
diff --git a/integration/bundle/clusters_test.go b/integration/bundle/clusters_test.go
index 449206208..b94b8365e 100644
--- a/integration/bundle/clusters_test.go
+++ b/integration/bundle/clusters_test.go
@@ -1,7 +1,6 @@
 package bundle_test
 
 import (
-	"fmt"
 	"testing"
 
 	"github.com/databricks/cli/integration/internal/acc"
@@ -29,7 +28,7 @@ func TestDeployBundleWithCluster(t *testing.T) {
 	t.Cleanup(func() {
 		destroyBundle(t, ctx, root)
 
-		cluster, err := wt.W.Clusters.GetByClusterName(ctx, fmt.Sprintf("test-cluster-%s", uniqueId))
+		cluster, err := wt.W.Clusters.GetByClusterName(ctx, "test-cluster-"+uniqueId)
 		if err != nil {
 			require.ErrorContains(t, err, "does not exist")
 		} else {
@@ -40,7 +39,7 @@ func TestDeployBundleWithCluster(t *testing.T) {
 	deployBundle(t, ctx, root)
 
 	// Cluster should exists after bundle deployment
-	cluster, err := wt.W.Clusters.GetByClusterName(ctx, fmt.Sprintf("test-cluster-%s", uniqueId))
+	cluster, err := wt.W.Clusters.GetByClusterName(ctx, "test-cluster-"+uniqueId)
 	require.NoError(t, err)
 	require.NotNil(t, cluster)
 
diff --git a/integration/bundle/dashboards_test.go b/integration/bundle/dashboards_test.go
index 83b4b8b03..a96b657f6 100644
--- a/integration/bundle/dashboards_test.go
+++ b/integration/bundle/dashboards_test.go
@@ -40,7 +40,7 @@ func TestDashboards(t *testing.T) {
 	// Load the dashboard by its ID and confirm its display name.
 	dashboard, err := wt.W.Lakeview.GetByDashboardId(ctx, oi.ResourceId)
 	require.NoError(t, err)
-	assert.Equal(t, fmt.Sprintf("test-dashboard-%s", uniqueID), dashboard.DisplayName)
+	assert.Equal(t, "test-dashboard-"+uniqueID, dashboard.DisplayName)
 
 	// Make an out of band modification to the dashboard and confirm that it is detected.
 	_, err = wt.W.Lakeview.Update(ctx, dashboards.UpdateDashboardRequest{
diff --git a/integration/bundle/deploy_to_shared_test.go b/integration/bundle/deploy_to_shared_test.go
index b4395f4c6..387d3c67a 100644
--- a/integration/bundle/deploy_to_shared_test.go
+++ b/integration/bundle/deploy_to_shared_test.go
@@ -1,7 +1,6 @@
 package bundle_test
 
 import (
-	"fmt"
 	"testing"
 
 	"github.com/databricks/cli/integration/internal/acc"
@@ -23,7 +22,7 @@ func TestDeployBasicToSharedWorkspacePath(t *testing.T) {
 		"unique_id":     uniqueId,
 		"node_type_id":  nodeTypeId,
 		"spark_version": defaultSparkVersion,
-		"root_path":     fmt.Sprintf("/Shared/%s", currentUser.UserName),
+		"root_path":     "/Shared/" + currentUser.UserName,
 	})
 
 	t.Cleanup(func() {
diff --git a/integration/bundle/empty_bundle_test.go b/integration/bundle/empty_bundle_test.go
index 1ab240d13..2c650cbef 100644
--- a/integration/bundle/empty_bundle_test.go
+++ b/integration/bundle/empty_bundle_test.go
@@ -1,7 +1,6 @@
 package bundle_test
 
 import (
-	"fmt"
 	"os"
 	"path/filepath"
 	"testing"
@@ -19,8 +18,7 @@ func TestEmptyBundleDeploy(t *testing.T) {
 	f, err := os.Create(filepath.Join(tmpDir, "databricks.yml"))
 	require.NoError(t, err)
 
-	bundleRoot := fmt.Sprintf(`bundle:
-  name: %s`, uuid.New().String())
+	bundleRoot := "bundle:\n  name: " + uuid.New().String()
 	_, err = f.WriteString(bundleRoot)
 	require.NoError(t, err)
 	f.Close()
diff --git a/integration/bundle/generate_job_test.go b/integration/bundle/generate_job_test.go
index b68bb7d61..f3c4c7829 100644
--- a/integration/bundle/generate_job_test.go
+++ b/integration/bundle/generate_job_test.go
@@ -2,10 +2,10 @@ package bundle_test
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"testing"
 
@@ -37,7 +37,7 @@ func TestGenerateFromExistingJobAndDeploy(t *testing.T) {
 
 	ctx = env.Set(ctx, "BUNDLE_ROOT", bundleRoot)
 	c := testcli.NewRunner(t, ctx, "bundle", "generate", "job",
-		"--existing-job-id", fmt.Sprint(jobId),
+		"--existing-job-id", strconv.FormatInt(jobId, 10),
 		"--config-dir", filepath.Join(bundleRoot, "resources"),
 		"--source-dir", filepath.Join(bundleRoot, "src"))
 	_, _, err := c.Run()
@@ -55,7 +55,7 @@ func TestGenerateFromExistingJobAndDeploy(t *testing.T) {
 	require.NoError(t, err)
 	generatedYaml := string(data)
 	require.Contains(t, generatedYaml, "notebook_task:")
-	require.Contains(t, generatedYaml, fmt.Sprintf("notebook_path: %s", filepath.Join("..", "src", "test.py")))
+	require.Contains(t, generatedYaml, "notebook_path: "+filepath.Join("..", "src", "test.py"))
 	require.Contains(t, generatedYaml, "task_key: test")
 	require.Contains(t, generatedYaml, "new_cluster:")
 	require.Contains(t, generatedYaml, "spark_version: 13.3.x-scala2.12")
diff --git a/integration/bundle/generate_pipeline_test.go b/integration/bundle/generate_pipeline_test.go
index 7843ec0c3..3565ab928 100644
--- a/integration/bundle/generate_pipeline_test.go
+++ b/integration/bundle/generate_pipeline_test.go
@@ -2,7 +2,6 @@ package bundle_test
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"path"
 	"path/filepath"
@@ -36,7 +35,7 @@ func TestGenerateFromExistingPipelineAndDeploy(t *testing.T) {
 
 	ctx = env.Set(ctx, "BUNDLE_ROOT", bundleRoot)
 	c := testcli.NewRunner(t, ctx, "bundle", "generate", "pipeline",
-		"--existing-pipeline-id", fmt.Sprint(pipelineId),
+		"--existing-pipeline-id", pipelineId,
 		"--config-dir", filepath.Join(bundleRoot, "resources"),
 		"--source-dir", filepath.Join(bundleRoot, "src"))
 	_, _, err := c.Run()
@@ -65,9 +64,9 @@ func TestGenerateFromExistingPipelineAndDeploy(t *testing.T) {
 
 	require.Contains(t, generatedYaml, "libraries:")
 	require.Contains(t, generatedYaml, "- notebook:")
-	require.Contains(t, generatedYaml, fmt.Sprintf("path: %s", filepath.Join("..", "src", "notebook.py")))
+	require.Contains(t, generatedYaml, "path: "+filepath.Join("..", "src", "notebook.py"))
 	require.Contains(t, generatedYaml, "- file:")
-	require.Contains(t, generatedYaml, fmt.Sprintf("path: %s", filepath.Join("..", "src", "test.py")))
+	require.Contains(t, generatedYaml, "path: "+filepath.Join("..", "src", "test.py"))
 
 	deployBundle(t, ctx, bundleRoot)
 
diff --git a/integration/bundle/init_test.go b/integration/bundle/init_test.go
index f5c263ca3..87a3e30e5 100644
--- a/integration/bundle/init_test.go
+++ b/integration/bundle/init_test.go
@@ -66,7 +66,7 @@ func TestBundleInitOnMlopsStacks(t *testing.T) {
 
 	// Assert that the README.md file was created
 	contents := testutil.ReadFile(t, filepath.Join(tmpDir2, "repo_name", projectName, "README.md"))
-	assert.Contains(t, contents, fmt.Sprintf("# %s", projectName))
+	assert.Contains(t, contents, "# "+projectName)
 
 	// Validate the stack
 	testutil.Chdir(t, filepath.Join(tmpDir2, "repo_name", projectName))
diff --git a/integration/cmd/auth/describe_test.go b/integration/cmd/auth/describe_test.go
index 41288dce6..f592bc276 100644
--- a/integration/cmd/auth/describe_test.go
+++ b/integration/cmd/auth/describe_test.go
@@ -2,7 +2,6 @@ package auth_test
 
 import (
 	"context"
-	"fmt"
 	"testing"
 
 	"github.com/databricks/cli/internal/testcli"
@@ -21,14 +20,14 @@ func TestAuthDescribeSuccess(t *testing.T) {
 	require.NoError(t, err)
 
 	require.NotEmpty(t, outStr)
-	require.Contains(t, outStr, fmt.Sprintf("Host: %s", w.Config.Host))
+	require.Contains(t, outStr, "Host: "+w.Config.Host)
 
 	me, err := w.CurrentUser.Me(context.Background())
 	require.NoError(t, err)
-	require.Contains(t, outStr, fmt.Sprintf("User: %s", me.UserName))
-	require.Contains(t, outStr, fmt.Sprintf("Authenticated with: %s", w.Config.AuthType))
+	require.Contains(t, outStr, "User: "+me.UserName)
+	require.Contains(t, outStr, "Authenticated with: "+w.Config.AuthType)
 	require.Contains(t, outStr, "Current configuration:")
-	require.Contains(t, outStr, fmt.Sprintf("✓ host: %s", w.Config.Host))
+	require.Contains(t, outStr, "✓ host: "+w.Config.Host)
 	require.Contains(t, outStr, "✓ profile: default")
 }
 
@@ -47,6 +46,6 @@ func TestAuthDescribeFailure(t *testing.T) {
 	w, err := databricks.NewWorkspaceClient(&databricks.Config{})
 	require.NoError(t, err)
 
-	require.Contains(t, outStr, fmt.Sprintf("✓ host: %s", w.Config.Host))
+	require.Contains(t, outStr, "✓ host: "+w.Config.Host)
 	require.Contains(t, outStr, "✓ profile: nonexistent (from --profile flag)")
 }
diff --git a/integration/cmd/fs/completion_test.go b/integration/cmd/fs/completion_test.go
index 88ce2fcc1..b13bf9d60 100644
--- a/integration/cmd/fs/completion_test.go
+++ b/integration/cmd/fs/completion_test.go
@@ -2,7 +2,6 @@ package fs_test
 
 import (
 	"context"
-	"fmt"
 	"strings"
 	"testing"
 
@@ -24,6 +23,6 @@ func TestFsCompletion(t *testing.T) {
 	setupCompletionFile(t, f)
 
 	stdout, _ := testcli.RequireSuccessfulRun(t, ctx, "__complete", "fs", "ls", tmpDir+"/")
-	expectedOutput := fmt.Sprintf("%s/dir1/\n:2\n", tmpDir)
+	expectedOutput := tmpDir + "/dir1/\n:2\n"
 	assert.Equal(t, expectedOutput, stdout.String())
 }
diff --git a/integration/cmd/jobs/jobs_test.go b/integration/cmd/jobs/jobs_test.go
index b6bcfc5b3..7ebc135a3 100644
--- a/integration/cmd/jobs/jobs_test.go
+++ b/integration/cmd/jobs/jobs_test.go
@@ -3,7 +3,7 @@ package jobs_test
 import (
 	"context"
 	"encoding/json"
-	"fmt"
+	"strconv"
 	"testing"
 
 	"github.com/databricks/cli/internal/testcli"
@@ -20,5 +20,5 @@ func TestCreateJob(t *testing.T) {
 	var output map[string]int
 	err := json.Unmarshal(stdout.Bytes(), &output)
 	require.NoError(t, err)
-	testcli.RequireSuccessfulRun(t, ctx, "jobs", "delete", fmt.Sprint(output["job_id"]), "--log-level=debug")
+	testcli.RequireSuccessfulRun(t, ctx, "jobs", "delete", strconv.Itoa(output["job_id"]), "--log-level=debug")
 }
diff --git a/integration/cmd/sync/sync_test.go b/integration/cmd/sync/sync_test.go
index 6f58b7e42..632497054 100644
--- a/integration/cmd/sync/sync_test.go
+++ b/integration/cmd/sync/sync_test.go
@@ -151,10 +151,7 @@ func (a *syncTest) remoteFileContent(ctx context.Context, relativePath, expected
 	filePath := path.Join(a.remoteRoot, relativePath)
 
 	// Remove leading "/" so we can use it in the URL.
-	urlPath := fmt.Sprintf(
-		"/api/2.0/workspace-files/%s",
-		strings.TrimLeft(filePath, "/"),
-	)
+	urlPath := "/api/2.0/workspace-files/" + strings.TrimLeft(filePath, "/")
 
 	apiClient, err := client.New(a.w.Config)
 	require.NoError(a.t, err)
diff --git a/integration/cmd/workspace/workspace_test.go b/integration/cmd/workspace/workspace_test.go
index 4edbbfc83..c376a87d2 100644
--- a/integration/cmd/workspace/workspace_test.go
+++ b/integration/cmd/workspace/workspace_test.go
@@ -114,7 +114,7 @@ func TestExportDir(t *testing.T) {
 	require.NoError(t, err)
 
 	expectedLogs := strings.Join([]string{
-		fmt.Sprintf("Exporting files from %s", sourceDir),
+		"Exporting files from " + sourceDir,
 		fmt.Sprintf("%s -> %s", path.Join(sourceDir, "a/b/c/file-b"), filepath.Join(targetDir, "a/b/c/file-b")),
 		fmt.Sprintf("%s -> %s", path.Join(sourceDir, "file-a"), filepath.Join(targetDir, "file-a")),
 		fmt.Sprintf("%s -> %s", path.Join(sourceDir, "pyNotebook"), filepath.Join(targetDir, "pyNotebook.py")),
@@ -185,7 +185,7 @@ func TestImportDir(t *testing.T) {
 	stdout, stderr := testcli.RequireSuccessfulRun(t, ctx, "workspace", "import-dir", "./testdata/import_dir", targetDir, "--log-level=debug")
 
 	expectedLogs := strings.Join([]string{
-		fmt.Sprintf("Importing files from %s", "./testdata/import_dir"),
+		"Importing files from " + "./testdata/import_dir",
 		fmt.Sprintf("%s -> %s", filepath.FromSlash("a/b/c/file-b"), path.Join(targetDir, "a/b/c/file-b")),
 		fmt.Sprintf("%s -> %s", filepath.FromSlash("file-a"), path.Join(targetDir, "file-a")),
 		fmt.Sprintf("%s -> %s", filepath.FromSlash("jupyterNotebook.ipynb"), path.Join(targetDir, "jupyterNotebook")),
diff --git a/integration/internal/acc/fixtures.go b/integration/internal/acc/fixtures.go
index cd867fb3a..2367d228f 100644
--- a/integration/internal/acc/fixtures.go
+++ b/integration/internal/acc/fixtures.go
@@ -45,7 +45,7 @@ func TemporaryDbfsDir(t *WorkspaceT, name ...string) string {
 
 	// Prefix the name with "integration-test-" to make it easier to identify.
 	name = append([]string{"integration-test-"}, name...)
-	path := fmt.Sprintf("/tmp/%s", testutil.RandomName(name...))
+	path := "/tmp/" + testutil.RandomName(name...)
 
 	t.Logf("Creating DBFS directory %s", path)
 	err := t.W.Dbfs.MkdirsByPath(ctx, path)
diff --git a/integration/python/python_tasks_test.go b/integration/python/python_tasks_test.go
index 9411afb13..39b38f890 100644
--- a/integration/python/python_tasks_test.go
+++ b/integration/python/python_tasks_test.go
@@ -5,7 +5,6 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
-	"fmt"
 	"os"
 	"path"
 	"slices"
@@ -244,8 +243,8 @@ func prepareDBFSFiles(t *testing.T) *testFiles {
 	return &testFiles{
 		w:               w,
 		pyNotebookPath:  path.Join(baseDir, "test.py"),
-		sparkPythonPath: fmt.Sprintf("dbfs:%s", path.Join(baseDir, "spark.py")),
-		wheelPath:       fmt.Sprintf("dbfs:%s", path.Join(baseDir, "my_test_code-0.0.1-py3-none-any.whl")),
+		sparkPythonPath: "dbfs:" + path.Join(baseDir, "spark.py"),
+		wheelPath:       "dbfs:" + path.Join(baseDir, "my_test_code-0.0.1-py3-none-any.whl"),
 	}
 }
 
@@ -268,7 +267,7 @@ func GenerateNotebookTasks(notebookPath string, versions []string, nodeTypeId st
 	tasks := make([]jobs.SubmitTask, 0)
 	for i := range versions {
 		task := jobs.SubmitTask{
-			TaskKey: fmt.Sprintf("notebook_%s", strings.ReplaceAll(versions[i], ".", "_")),
+			TaskKey: "notebook_" + strings.ReplaceAll(versions[i], ".", "_"),
 			NotebookTask: &jobs.NotebookTask{
 				NotebookPath: notebookPath,
 			},
@@ -289,7 +288,7 @@ func GenerateSparkPythonTasks(notebookPath string, versions []string, nodeTypeId
 	tasks := make([]jobs.SubmitTask, 0)
 	for i := range versions {
 		task := jobs.SubmitTask{
-			TaskKey: fmt.Sprintf("spark_%s", strings.ReplaceAll(versions[i], ".", "_")),
+			TaskKey: "spark_" + strings.ReplaceAll(versions[i], ".", "_"),
 			SparkPythonTask: &jobs.SparkPythonTask{
 				PythonFile: notebookPath,
 			},
@@ -310,7 +309,7 @@ func GenerateWheelTasks(wheelPath string, versions []string, nodeTypeId string)
 	tasks := make([]jobs.SubmitTask, 0)
 	for i := range versions {
 		task := jobs.SubmitTask{
-			TaskKey: fmt.Sprintf("whl_%s", strings.ReplaceAll(versions[i], ".", "_")),
+			TaskKey: "whl_" + strings.ReplaceAll(versions[i], ".", "_"),
 			PythonWheelTask: &jobs.PythonWheelTask{
 				PackageName: "my_test_code",
 				EntryPoint:  "run",
diff --git a/libs/auth/oauth.go b/libs/auth/oauth.go
index 026c45468..1037a5a85 100644
--- a/libs/auth/oauth.go
+++ b/libs/auth/oauth.go
@@ -107,7 +107,7 @@ func (a *PersistentAuth) Load(ctx context.Context) (*oauth2.Token, error) {
 
 func (a *PersistentAuth) ProfileName() string {
 	if a.AccountID != "" {
-		return fmt.Sprintf("ACCOUNT-%s", a.AccountID)
+		return "ACCOUNT-" + a.AccountID
 	}
 	host := strings.TrimPrefix(a.Host, "https://")
 	split := strings.Split(host, ".")
@@ -210,12 +210,12 @@ func (a *PersistentAuth) oidcEndpoints(ctx context.Context) (*oauthAuthorization
 	prefix := a.key()
 	if a.AccountID != "" {
 		return &oauthAuthorizationServer{
-			AuthorizationEndpoint: fmt.Sprintf("%s/v1/authorize", prefix),
-			TokenEndpoint:         fmt.Sprintf("%s/v1/token", prefix),
+			AuthorizationEndpoint: prefix + "/v1/authorize",
+			TokenEndpoint:         prefix + "/v1/token",
 		}, nil
 	}
 	var oauthEndpoints oauthAuthorizationServer
-	oidc := fmt.Sprintf("%s/oidc/.well-known/oauth-authorization-server", prefix)
+	oidc := prefix + "/oidc/.well-known/oauth-authorization-server"
 	err := a.http.Do(ctx, "GET", oidc, httpclient.WithResponseUnmarshal(&oauthEndpoints))
 	if err != nil {
 		return nil, fmt.Errorf("fetch .well-known: %w", err)
@@ -247,7 +247,7 @@ func (a *PersistentAuth) oauth2Config(ctx context.Context) (*oauth2.Config, erro
 			TokenURL:  endpoints.TokenEndpoint,
 			AuthStyle: oauth2.AuthStyleInParams,
 		},
-		RedirectURL: fmt.Sprintf("http://%s", appRedirectAddr),
+		RedirectURL: "http://" + appRedirectAddr,
 		Scopes:      scopes,
 	}, nil
 }
@@ -258,7 +258,7 @@ func (a *PersistentAuth) oauth2Config(ctx context.Context) (*oauth2.Config, erro
 func (a *PersistentAuth) key() string {
 	a.Host = strings.TrimSuffix(a.Host, "/")
 	if !strings.HasPrefix(a.Host, "http") {
-		a.Host = fmt.Sprintf("https://%s", a.Host)
+		a.Host = "https://" + a.Host
 	}
 	if a.AccountID != "" {
 		return fmt.Sprintf("%s/oidc/accounts/%s", a.Host, a.AccountID)
diff --git a/libs/auth/oauth_test.go b/libs/auth/oauth_test.go
index 837ff4fee..6c3b9bf47 100644
--- a/libs/auth/oauth_test.go
+++ b/libs/auth/oauth_test.go
@@ -112,7 +112,7 @@ func TestLoadRefresh(t *testing.T) {
 		},
 	}.ApplyClient(t, func(ctx context.Context, c *client.DatabricksClient) {
 		ctx = useInsecureOAuthHttpClientForTests(ctx)
-		expectedKey := fmt.Sprintf("%s/oidc/accounts/xyz", c.Config.Host)
+		expectedKey := c.Config.Host + "/oidc/accounts/xyz"
 		p := &PersistentAuth{
 			Host:      c.Config.Host,
 			AccountID: "xyz",
@@ -149,7 +149,7 @@ func TestChallenge(t *testing.T) {
 		},
 	}.ApplyClient(t, func(ctx context.Context, c *client.DatabricksClient) {
 		ctx = useInsecureOAuthHttpClientForTests(ctx)
-		expectedKey := fmt.Sprintf("%s/oidc/accounts/xyz", c.Config.Host)
+		expectedKey := c.Config.Host + "/oidc/accounts/xyz"
 
 		browserOpened := make(chan string)
 		p := &PersistentAuth{
diff --git a/libs/cmdio/error_event.go b/libs/cmdio/error_event.go
index 933f9d0d0..62897995b 100644
--- a/libs/cmdio/error_event.go
+++ b/libs/cmdio/error_event.go
@@ -1,13 +1,11 @@
 package cmdio
 
-import "fmt"
-
 type ErrorEvent struct {
 	Error string `json:"error"`
 }
 
 func (event *ErrorEvent) String() string {
-	return fmt.Sprintf("Error: %s", event.Error)
+	return "Error: " + event.Error
 }
 
 func (event *ErrorEvent) IsInplaceSupported() bool {
diff --git a/libs/cmdio/logger.go b/libs/cmdio/logger.go
index 7bc95e9a5..7edad5bf0 100644
--- a/libs/cmdio/logger.go
+++ b/libs/cmdio/logger.go
@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -124,7 +125,7 @@ func splitAtLastNewLine(s string) (string, string) {
 
 func (l *Logger) AskSelect(question string, choices []string) (string, error) {
 	if l.Mode == flags.ModeJson {
-		return "", fmt.Errorf("question prompts are not supported in json mode")
+		return "", errors.New("question prompts are not supported in json mode")
 	}
 
 	// Promptui does not support multiline prompts. So we split the question.
@@ -140,7 +141,7 @@ func (l *Logger) AskSelect(question string, choices []string) (string, error) {
 		HideHelp: true,
 		Templates: &promptui.SelectTemplates{
 			Label:    "{{.}}: ",
-			Selected: fmt.Sprintf("%s: {{.}}", last),
+			Selected: last + ": {{.}}",
 		},
 	}
 
@@ -153,7 +154,7 @@ func (l *Logger) AskSelect(question string, choices []string) (string, error) {
 
 func (l *Logger) Ask(question, defaultVal string) (string, error) {
 	if l.Mode == flags.ModeJson {
-		return "", fmt.Errorf("question prompts are not supported in json mode")
+		return "", errors.New("question prompts are not supported in json mode")
 	}
 
 	// Add default value to question prompt.
diff --git a/libs/databrickscfg/cfgpickers/clusters.go b/libs/databrickscfg/cfgpickers/clusters.go
index 6ae7d99c6..e27d13690 100644
--- a/libs/databrickscfg/cfgpickers/clusters.go
+++ b/libs/databrickscfg/cfgpickers/clusters.go
@@ -33,7 +33,7 @@ func GetRuntimeVersion(cluster compute.ClusterDetails) (string, bool) {
 		match = dbrSnapshotVersionRegex.FindStringSubmatch(cluster.SparkVersion)
 		if len(match) > 1 {
 			// we return 14.999 for 14.x-snapshot for semver.Compare() to work properly
-			return fmt.Sprintf("%s.999", match[1]), true
+			return match[1] + ".999", true
 		}
 		return "", false
 	}
diff --git a/libs/databrickscfg/loader.go b/libs/databrickscfg/loader.go
index 12a516c59..84c8398bf 100644
--- a/libs/databrickscfg/loader.go
+++ b/libs/databrickscfg/loader.go
@@ -19,7 +19,7 @@ var errNoMatchingProfiles = errors.New("no matching config profiles found")
 type errMultipleProfiles []string
 
 func (e errMultipleProfiles) Error() string {
-	return fmt.Sprintf("multiple profiles matched: %s", strings.Join(e, ", "))
+	return "multiple profiles matched: " + strings.Join(e, ", ")
 }
 
 func findMatchingProfile(configFile *config.File, matcher func(*ini.Section) bool) (*ini.Section, error) {
diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go
index 31cd8b6e3..ee26d5afc 100644
--- a/libs/dyn/convert/normalize.go
+++ b/libs/dyn/convert/normalize.go
@@ -97,7 +97,7 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen
 				if !pv.IsAnchor() {
 					diags = diags.Append(diag.Diagnostic{
 						Severity: diag.Warning,
-						Summary:  fmt.Sprintf("unknown field: %s", pk.MustString()),
+						Summary:  "unknown field: " + pk.MustString(),
 						// Show all locations the unknown field is defined at.
 						Locations: pk.Locations(),
 						Paths:     []dyn.Path{path},
diff --git a/libs/dyn/jsonloader/json.go b/libs/dyn/jsonloader/json.go
index 3f2dc859f..708fc401f 100644
--- a/libs/dyn/jsonloader/json.go
+++ b/libs/dyn/jsonloader/json.go
@@ -3,6 +3,7 @@ package jsonloader
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 
@@ -20,7 +21,7 @@ func LoadJSON(data []byte, source string) (dyn.Value, error) {
 	value, err := decodeValue(decoder, &offset)
 	if err != nil {
 		if err == io.EOF {
-			err = fmt.Errorf("unexpected end of JSON input")
+			err = errors.New("unexpected end of JSON input")
 		}
 		return dyn.InvalidValue, fmt.Errorf("error decoding JSON at %s: %v", value.Location(), err)
 	}
@@ -57,7 +58,7 @@ func decodeValue(decoder *json.Decoder, o *Offset) (dyn.Value, error) {
 				}
 				key, ok := keyToken.(string)
 				if !ok {
-					return invalidValueWithLocation(decoder, o), fmt.Errorf("expected string for object key")
+					return invalidValueWithLocation(decoder, o), errors.New("expected string for object key")
 				}
 
 				// Get the offset of the key by subtracting the length of the key and the '"' character
diff --git a/libs/dyn/location.go b/libs/dyn/location.go
index 961d2f121..d2b2ad596 100644
--- a/libs/dyn/location.go
+++ b/libs/dyn/location.go
@@ -1,6 +1,7 @@
 package dyn
 
 import (
+	"errors"
 	"fmt"
 	"path/filepath"
 )
@@ -17,7 +18,7 @@ func (l Location) String() string {
 
 func (l Location) Directory() (string, error) {
 	if l.File == "" {
-		return "", fmt.Errorf("no file in location")
+		return "", errors.New("no file in location")
 	}
 
 	return filepath.Dir(l.File), nil
diff --git a/libs/dyn/mapping_test.go b/libs/dyn/mapping_test.go
index 67144ae55..d0347d22a 100644
--- a/libs/dyn/mapping_test.go
+++ b/libs/dyn/mapping_test.go
@@ -1,7 +1,7 @@
 package dyn_test
 
 import (
-	"fmt"
+	"strconv"
 	"testing"
 
 	"github.com/databricks/cli/libs/dyn"
@@ -186,13 +186,13 @@ func TestMappingClone(t *testing.T) {
 func TestMappingMerge(t *testing.T) {
 	var m1 dyn.Mapping
 	for i := range 10 {
-		err := m1.Set(dyn.V(fmt.Sprintf("%d", i)), dyn.V(i))
+		err := m1.Set(dyn.V(strconv.Itoa(i)), dyn.V(i))
 		require.NoError(t, err)
 	}
 
 	var m2 dyn.Mapping
 	for i := 5; i < 15; i++ {
-		err := m2.Set(dyn.V(fmt.Sprintf("%d", i)), dyn.V(i))
+		err := m2.Set(dyn.V(strconv.Itoa(i)), dyn.V(i))
 		require.NoError(t, err)
 	}
 
diff --git a/libs/dyn/merge/override_test.go b/libs/dyn/merge/override_test.go
index ea161d27c..d9d3f3983 100644
--- a/libs/dyn/merge/override_test.go
+++ b/libs/dyn/merge/override_test.go
@@ -1,7 +1,7 @@
 package merge
 
 import (
-	"fmt"
+	"errors"
 	"testing"
 	"time"
 
@@ -373,7 +373,7 @@ func TestOverride_Primitive(t *testing.T) {
 
 		if modified {
 			t.Run(tc.name+" - visitor has error", func(t *testing.T) {
-				_, visitor := createVisitor(visitorOpts{error: fmt.Errorf("unexpected change in test")})
+				_, visitor := createVisitor(visitorOpts{error: errors.New("unexpected change in test")})
 				_, err := override(dyn.EmptyPath, tc.left, tc.right, visitor)
 
 				assert.EqualError(t, err, "unexpected change in test")
diff --git a/libs/dyn/path_string_test.go b/libs/dyn/path_string_test.go
index 0d64bf110..eb1816d7d 100644
--- a/libs/dyn/path_string_test.go
+++ b/libs/dyn/path_string_test.go
@@ -1,7 +1,7 @@
 package dyn_test
 
 import (
-	"fmt"
+	"errors"
 	"testing"
 
 	. "github.com/databricks/cli/libs/dyn"
@@ -52,31 +52,31 @@ func TestNewPathFromString(t *testing.T) {
 		},
 		{
 			input: "foo[123",
-			err:   fmt.Errorf("invalid path: foo[123"),
+			err:   errors.New("invalid path: foo[123"),
 		},
 		{
 			input: "foo[123]]",
-			err:   fmt.Errorf("invalid path: foo[123]]"),
+			err:   errors.New("invalid path: foo[123]]"),
 		},
 		{
 			input: "foo[[123]",
-			err:   fmt.Errorf("invalid path: foo[[123]"),
+			err:   errors.New("invalid path: foo[[123]"),
 		},
 		{
 			input: "foo[[123]]",
-			err:   fmt.Errorf("invalid path: foo[[123]]"),
+			err:   errors.New("invalid path: foo[[123]]"),
 		},
 		{
 			input: "foo[foo]",
-			err:   fmt.Errorf("invalid path: foo[foo]"),
+			err:   errors.New("invalid path: foo[foo]"),
 		},
 		{
 			input: "foo..bar",
-			err:   fmt.Errorf("invalid path: foo..bar"),
+			err:   errors.New("invalid path: foo..bar"),
 		},
 		{
 			input: "foo.bar.",
-			err:   fmt.Errorf("invalid path: foo.bar."),
+			err:   errors.New("invalid path: foo.bar."),
 		},
 		{
 			// Every component may have a leading dot.
@@ -86,7 +86,7 @@ func TestNewPathFromString(t *testing.T) {
 		{
 			// But after an index there must be a dot.
 			input: "foo[1]bar",
-			err:   fmt.Errorf("invalid path: foo[1]bar"),
+			err:   errors.New("invalid path: foo[1]bar"),
 		},
 	} {
 		p, err := NewPathFromString(tc.input)
diff --git a/libs/dyn/visit_map_test.go b/libs/dyn/visit_map_test.go
index 3c2908c4b..ad091743d 100644
--- a/libs/dyn/visit_map_test.go
+++ b/libs/dyn/visit_map_test.go
@@ -1,6 +1,7 @@
 package dyn_test
 
 import (
+	"errors"
 	"fmt"
 	"testing"
 
@@ -71,7 +72,7 @@ func TestMapFuncOnMap(t *testing.T) {
 	}, vbar.AsAny())
 
 	// Return error from map function.
-	ref := fmt.Errorf("error")
+	ref := errors.New("error")
 	verr, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Key("foo")), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
 		return dyn.InvalidValue, ref
 	})
@@ -137,7 +138,7 @@ func TestMapFuncOnSequence(t *testing.T) {
 	assert.Equal(t, []any{42, 45}, v1.AsAny())
 
 	// Return error from map function.
-	ref := fmt.Errorf("error")
+	ref := errors.New("error")
 	verr, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Index(0)), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
 		return dyn.InvalidValue, ref
 	})
@@ -211,7 +212,7 @@ func TestMapForeachOnMapError(t *testing.T) {
 	})
 
 	// Check that an error from the map function propagates.
-	ref := fmt.Errorf("error")
+	ref := errors.New("error")
 	_, err := dyn.Map(vin, ".", dyn.Foreach(func(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
 		return dyn.InvalidValue, ref
 	}))
@@ -255,7 +256,7 @@ func TestMapForeachOnSequenceError(t *testing.T) {
 	})
 
 	// Check that an error from the map function propagates.
-	ref := fmt.Errorf("error")
+	ref := errors.New("error")
 	_, err := dyn.Map(vin, ".", dyn.Foreach(func(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
 		return dyn.InvalidValue, ref
 	}))
diff --git a/libs/dyn/yamlsaver/saver.go b/libs/dyn/yamlsaver/saver.go
index 7398e2594..a7838ff36 100644
--- a/libs/dyn/yamlsaver/saver.go
+++ b/libs/dyn/yamlsaver/saver.go
@@ -123,9 +123,9 @@ func (s *saver) toYamlNodeWithStyle(v dyn.Value, style yaml.Style) (*yaml.Node,
 		}
 		return &yaml.Node{Kind: yaml.ScalarNode, Value: v.MustString(), Style: style}, nil
 	case dyn.KindBool:
-		return &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprint(v.MustBool()), Style: style}, nil
+		return &yaml.Node{Kind: yaml.ScalarNode, Value: strconv.FormatBool(v.MustBool()), Style: style}, nil
 	case dyn.KindInt:
-		return &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprint(v.MustInt()), Style: style}, nil
+		return &yaml.Node{Kind: yaml.ScalarNode, Value: strconv.FormatInt(v.MustInt(), 10), Style: style}, nil
 	case dyn.KindFloat:
 		return &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprint(v.MustFloat()), Style: style}, nil
 	case dyn.KindTime:
diff --git a/libs/errs/aggregate_test.go b/libs/errs/aggregate_test.go
index 1af57e099..216276a06 100644
--- a/libs/errs/aggregate_test.go
+++ b/libs/errs/aggregate_test.go
@@ -1,16 +1,16 @@
 package errs
 
 import (
-	"fmt"
+	"errors"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
 func TestFromManyErrors(t *testing.T) {
-	e1 := fmt.Errorf("Error 1")
-	e2 := fmt.Errorf("Error 2")
-	e3 := fmt.Errorf("Error 3")
+	e1 := errors.New("Error 1")
+	e2 := errors.New("Error 2")
+	e3 := errors.New("Error 3")
 	err := FromMany(e1, e2, e3)
 
 	assert.ErrorIs(t, err, e1)
@@ -23,9 +23,9 @@ Error 3`, err.Error())
 }
 
 func TestFromManyErrorsWihtNil(t *testing.T) {
-	e1 := fmt.Errorf("Error 1")
+	e1 := errors.New("Error 1")
 	var e2 error = nil
-	e3 := fmt.Errorf("Error 3")
+	e3 := errors.New("Error 3")
 	err := FromMany(e1, e2, e3)
 
 	assert.ErrorIs(t, err, e1)
diff --git a/libs/exec/shell_cmd.go b/libs/exec/shell_cmd.go
index 164d09739..057ed06a4 100644
--- a/libs/exec/shell_cmd.go
+++ b/libs/exec/shell_cmd.go
@@ -2,7 +2,6 @@ package exec
 
 import (
 	"errors"
-	"fmt"
 	osexec "os/exec"
 )
 
@@ -18,7 +17,7 @@ func (s cmdShell) prepare(command string) (*execContext, error) {
 
 	return &execContext{
 		executable: s.executable,
-		args:       []string{"/D", "/E:ON", "/V:OFF", "/S", "/C", fmt.Sprintf(`CALL %s`, filename)},
+		args:       []string{"/D", "/E:ON", "/V:OFF", "/S", "/C", "CALL " + filename},
 		scriptFile: filename,
 	}, nil
 }
diff --git a/libs/fakefs/fakefs.go b/libs/fakefs/fakefs.go
index a8d5eb873..050ee2d6e 100644
--- a/libs/fakefs/fakefs.go
+++ b/libs/fakefs/fakefs.go
@@ -1,12 +1,12 @@
 package fakefs
 
 import (
-	"fmt"
+	"errors"
 	"io/fs"
 	"time"
 )
 
-var ErrNotImplemented = fmt.Errorf("not implemented")
+var ErrNotImplemented = errors.New("not implemented")
 
 // DirEntry is a fake implementation of [fs.DirEntry].
 type DirEntry struct {
diff --git a/libs/filer/fake_filer.go b/libs/filer/fake_filer.go
index 76b8bcd94..1e1cbd985 100644
--- a/libs/filer/fake_filer.go
+++ b/libs/filer/fake_filer.go
@@ -2,7 +2,7 @@ package filer
 
 import (
 	"context"
-	"fmt"
+	"errors"
 	"io"
 	"io/fs"
 	"path"
@@ -17,7 +17,7 @@ type FakeFiler struct {
 }
 
 func (f *FakeFiler) Write(ctx context.Context, p string, reader io.Reader, mode ...WriteMode) error {
-	return fmt.Errorf("not implemented")
+	return errors.New("not implemented")
 }
 
 func (f *FakeFiler) Read(ctx context.Context, p string) (io.ReadCloser, error) {
@@ -30,7 +30,7 @@ func (f *FakeFiler) Read(ctx context.Context, p string) (io.ReadCloser, error) {
 }
 
 func (f *FakeFiler) Delete(ctx context.Context, p string, mode ...DeleteMode) error {
-	return fmt.Errorf("not implemented")
+	return errors.New("not implemented")
 }
 
 func (f *FakeFiler) ReadDir(ctx context.Context, p string) ([]fs.DirEntry, error) {
@@ -59,7 +59,7 @@ func (f *FakeFiler) ReadDir(ctx context.Context, p string) ([]fs.DirEntry, error
 }
 
 func (f *FakeFiler) Mkdir(ctx context.Context, path string) error {
-	return fmt.Errorf("not implemented")
+	return errors.New("not implemented")
 }
 
 func (f *FakeFiler) Stat(ctx context.Context, path string) (fs.FileInfo, error) {
diff --git a/libs/filer/filer.go b/libs/filer/filer.go
index 83dc560cb..372c82929 100644
--- a/libs/filer/filer.go
+++ b/libs/filer/filer.go
@@ -2,7 +2,6 @@ package filer
 
 import (
 	"context"
-	"fmt"
 	"io"
 	"io/fs"
 )
@@ -36,7 +35,7 @@ type FileAlreadyExistsError struct {
 }
 
 func (err FileAlreadyExistsError) Error() string {
-	return fmt.Sprintf("file already exists: %s", err.path)
+	return "file already exists: " + err.path
 }
 
 func (err FileAlreadyExistsError) Is(other error) bool {
@@ -52,7 +51,7 @@ func (err FileDoesNotExistError) Is(other error) bool {
 }
 
 func (err FileDoesNotExistError) Error() string {
-	return fmt.Sprintf("file does not exist: %s", err.path)
+	return "file does not exist: " + err.path
 }
 
 type NoSuchDirectoryError struct {
@@ -60,7 +59,7 @@ type NoSuchDirectoryError struct {
 }
 
 func (err NoSuchDirectoryError) Error() string {
-	return fmt.Sprintf("no such directory: %s", err.path)
+	return "no such directory: " + err.path
 }
 
 func (err NoSuchDirectoryError) Is(other error) bool {
@@ -72,7 +71,7 @@ type NotADirectory struct {
 }
 
 func (err NotADirectory) Error() string {
-	return fmt.Sprintf("not a directory: %s", err.path)
+	return "not a directory: " + err.path
 }
 
 func (err NotADirectory) Is(other error) bool {
@@ -84,7 +83,7 @@ type NotAFile struct {
 }
 
 func (err NotAFile) Error() string {
-	return fmt.Sprintf("not a file: %s", err.path)
+	return "not a file: " + err.path
 }
 
 func (err NotAFile) Is(other error) bool {
@@ -96,7 +95,7 @@ type DirectoryNotEmptyError struct {
 }
 
 func (err DirectoryNotEmptyError) Error() string {
-	return fmt.Sprintf("directory not empty: %s", err.path)
+	return "directory not empty: " + err.path
 }
 
 func (err DirectoryNotEmptyError) Is(other error) bool {
@@ -118,7 +117,7 @@ type PermissionError struct {
 }
 
 func (err PermissionError) Error() string {
-	return fmt.Sprintf("access denied: %s", err.path)
+	return "access denied: " + err.path
 }
 
 func (err PermissionError) Is(other error) bool {
diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go
index 7ea1d0f03..98a534684 100644
--- a/libs/filer/files_client.go
+++ b/libs/filer/files_client.go
@@ -116,10 +116,7 @@ func (w *FilesClient) urlPath(name string) (string, string, error) {
 	}
 
 	// The user specified part of the path must be escaped.
-	urlPath := fmt.Sprintf(
-		"/api/2.0/fs/files/%s",
-		url.PathEscape(strings.TrimLeft(absPath, "/")),
-	)
+	urlPath := "/api/2.0/fs/files/" + url.PathEscape(strings.TrimLeft(absPath, "/"))
 
 	return absPath, urlPath, nil
 }
diff --git a/libs/filer/workspace_files_cache_test.go b/libs/filer/workspace_files_cache_test.go
index 8983c5982..a73f415c1 100644
--- a/libs/filer/workspace_files_cache_test.go
+++ b/libs/filer/workspace_files_cache_test.go
@@ -2,7 +2,7 @@ package filer
 
 import (
 	"context"
-	"fmt"
+	"errors"
 	"io"
 	"io/fs"
 	"testing"
@@ -11,7 +11,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-var errNotImplemented = fmt.Errorf("not implemented")
+var errNotImplemented = errors.New("not implemented")
 
 type cacheTestFiler struct {
 	calls int
diff --git a/libs/flags/json_flag_test.go b/libs/flags/json_flag_test.go
index 956a3541c..4bebf8b68 100644
--- a/libs/flags/json_flag_test.go
+++ b/libs/flags/json_flag_test.go
@@ -1,7 +1,6 @@
 package flags
 
 import (
-	"fmt"
 	"os"
 	"path"
 	"testing"
@@ -68,7 +67,7 @@ func TestJsonFlagFile(t *testing.T) {
 		fpath = f.Name()
 	}
 
-	err := body.Set(fmt.Sprintf("@%s", fpath))
+	err := body.Set("@" + fpath)
 	require.NoError(t, err)
 
 	diags := body.Unmarshal(&request)
diff --git a/libs/flags/output.go b/libs/flags/output.go
index 17da144bd..e0c799131 100644
--- a/libs/flags/output.go
+++ b/libs/flags/output.go
@@ -1,6 +1,7 @@
 package flags
 
 import (
+	"errors"
 	"fmt"
 	"strings"
 
@@ -25,7 +26,7 @@ func (f *Output) Set(s string) error {
 	case `json`, `text`:
 		*f = Output(lower)
 	default:
-		return fmt.Errorf("accepted arguments are json and text")
+		return errors.New("accepted arguments are json and text")
 	}
 	return nil
 }
diff --git a/libs/git/reference.go b/libs/git/reference.go
index e1126d4f2..6001d70de 100644
--- a/libs/git/reference.go
+++ b/libs/git/reference.go
@@ -13,8 +13,8 @@ import (
 type ReferenceType string
 
 var (
-	ErrNotAReferencePointer = fmt.Errorf("HEAD does not point to another reference")
-	ErrNotABranch           = fmt.Errorf("HEAD is not a reference to a git branch")
+	ErrNotAReferencePointer = errors.New("HEAD does not point to another reference")
+	ErrNotABranch           = errors.New("HEAD is not a reference to a git branch")
 )
 
 const (
diff --git a/libs/git/repository_test.go b/libs/git/repository_test.go
index 857df65a9..58a540190 100644
--- a/libs/git/repository_test.go
+++ b/libs/git/repository_test.go
@@ -1,7 +1,6 @@
 package git
 
 import (
-	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -96,8 +95,7 @@ func (testRepo *testRepository) addOriginUrl(url string) {
 	defer f.Close()
 
 	_, err = f.WriteString(
-		fmt.Sprintf(`[remote "origin"]
-	url = %s`, url))
+		"[remote \"origin\"]\n\turl = " + url)
 	require.NoError(testRepo.t, err)
 
 	// reload config to reflect the remote url
diff --git a/libs/git/worktree_test.go b/libs/git/worktree_test.go
index 3d620c483..072a9d348 100644
--- a/libs/git/worktree_test.go
+++ b/libs/git/worktree_test.go
@@ -53,12 +53,12 @@ func TestWorktreeResolveGitDir(t *testing.T) {
 	writeGitCommonDir(t, dir, "../..")
 
 	t.Run("relative", func(t *testing.T) {
-		writeGitDir(t, dir, fmt.Sprintf("gitdir: %s", "../.git/worktrees/my_worktree"))
+		writeGitDir(t, dir, "gitdir: "+"../.git/worktrees/my_worktree")
 		verifyCorrectDirs(t, dir)
 	})
 
 	t.Run("absolute", func(t *testing.T) {
-		writeGitDir(t, dir, fmt.Sprintf("gitdir: %s", filepath.Join(dir, ".git/worktrees/my_worktree")))
+		writeGitDir(t, dir, "gitdir: "+filepath.Join(dir, ".git/worktrees/my_worktree"))
 		verifyCorrectDirs(t, dir)
 	})
 
@@ -77,7 +77,7 @@ func TestWorktreeResolveGitDir(t *testing.T) {
 
 func TestWorktreeResolveCommonDir(t *testing.T) {
 	dir := setupWorktree(t)
-	writeGitDir(t, dir, fmt.Sprintf("gitdir: %s", "../.git/worktrees/my_worktree"))
+	writeGitDir(t, dir, "gitdir: "+"../.git/worktrees/my_worktree")
 
 	t.Run("relative", func(t *testing.T) {
 		writeGitCommonDir(t, dir, "../..")
diff --git a/libs/jsonschema/instance.go b/libs/jsonschema/instance.go
index 4440a2fe2..eb36822a0 100644
--- a/libs/jsonschema/instance.go
+++ b/libs/jsonschema/instance.go
@@ -2,6 +2,7 @@ package jsonschema
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"slices"
@@ -149,7 +150,7 @@ func (s *Schema) validateAnyOf(instance map[string]any) error {
 	// According to the JSON schema RFC, anyOf must contain at least one schema.
 	// https://json-schema.org/draft/2020-12/json-schema-core
 	if len(s.AnyOf) == 0 {
-		return fmt.Errorf("anyOf must contain at least one schema")
+		return errors.New("anyOf must contain at least one schema")
 	}
 
 	for _, anyOf := range s.AnyOf {
@@ -158,5 +159,5 @@ func (s *Schema) validateAnyOf(instance map[string]any) error {
 			return nil
 		}
 	}
-	return fmt.Errorf("instance does not match any of the schemas in anyOf")
+	return errors.New("instance does not match any of the schemas in anyOf")
 }
diff --git a/libs/jsonschema/utils.go b/libs/jsonschema/utils.go
index ff9b88312..bc9339cae 100644
--- a/libs/jsonschema/utils.go
+++ b/libs/jsonschema/utils.go
@@ -150,7 +150,7 @@ func (e patternMatchError) Error() string {
 	// If custom user error message is defined, return error with the custom message
 	msg := e.FailureMessage
 	if msg == "" {
-		msg = fmt.Sprintf("Expected to match regex pattern: %s", e.Pattern)
+		msg = "Expected to match regex pattern: " + e.Pattern
 	}
 	return fmt.Sprintf("invalid value for %s: %q. %s", e.PropertyName, e.PropertyValue, msg)
 }
diff --git a/libs/locker/locker.go b/libs/locker/locker.go
index eb59c9f74..aadc50b58 100644
--- a/libs/locker/locker.go
+++ b/libs/locker/locker.go
@@ -116,14 +116,14 @@ func (locker *Locker) assertLockHeld(ctx context.Context) error {
 // idempotent function since overwrite is set to true
 func (locker *Locker) Write(ctx context.Context, pathToFile string, content []byte) error {
 	if !locker.Active {
-		return fmt.Errorf("failed to put file. deploy lock not held")
+		return errors.New("failed to put file. deploy lock not held")
 	}
 	return locker.filer.Write(ctx, pathToFile, bytes.NewReader(content), filer.OverwriteIfExists, filer.CreateParentDirectories)
 }
 
 func (locker *Locker) Read(ctx context.Context, path string) (io.ReadCloser, error) {
 	if !locker.Active {
-		return nil, fmt.Errorf("failed to get file. deploy lock not held")
+		return nil, errors.New("failed to get file. deploy lock not held")
 	}
 	return locker.filer.Read(ctx, path)
 }
@@ -173,7 +173,7 @@ func (locker *Locker) Lock(ctx context.Context, isForced bool) error {
 
 func (locker *Locker) Unlock(ctx context.Context, opts ...UnlockOption) error {
 	if !locker.Active {
-		return fmt.Errorf("unlock called when lock is not held")
+		return errors.New("unlock called when lock is not held")
 	}
 
 	// if allowLockFileNotExist is set, do not throw an error if the lock file does
diff --git a/libs/process/background_test.go b/libs/process/background_test.go
index 7843375cf..5cc810f5d 100644
--- a/libs/process/background_test.go
+++ b/libs/process/background_test.go
@@ -4,7 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"context"
-	"fmt"
+	"errors"
 	"os/exec"
 	"strings"
 	"testing"
@@ -101,7 +101,7 @@ func TestBackgroundFails(t *testing.T) {
 func TestBackgroundFailsOnOption(t *testing.T) {
 	ctx := context.Background()
 	_, err := Background(ctx, []string{"ls", "/dev/null/x"}, func(_ context.Context, c *exec.Cmd) error {
-		return fmt.Errorf("nope")
+		return errors.New("nope")
 	})
 	assert.EqualError(t, err, "nope")
 }
diff --git a/libs/process/stub.go b/libs/process/stub.go
index 8ab6fd705..528489098 100644
--- a/libs/process/stub.go
+++ b/libs/process/stub.go
@@ -168,7 +168,7 @@ func (s *processStub) run(cmd *exec.Cmd) error {
 	}
 	var zeroStub reponseStub
 	if s.reponseStub == zeroStub {
-		return fmt.Errorf("no default process stub")
+		return errors.New("no default process stub")
 	}
 	err := s.reponseStub.err
 	if s.reponseStub.stdout != "" {
diff --git a/libs/process/stub_test.go b/libs/process/stub_test.go
index 81afa3a89..158e8b3a6 100644
--- a/libs/process/stub_test.go
+++ b/libs/process/stub_test.go
@@ -2,7 +2,7 @@ package process_test
 
 import (
 	"context"
-	"fmt"
+	"errors"
 	"os/exec"
 	"testing"
 
@@ -32,7 +32,7 @@ func TestStubOutput(t *testing.T) {
 func TestStubFailure(t *testing.T) {
 	ctx := context.Background()
 	ctx, stub := process.WithStub(ctx)
-	stub.WithFailure(fmt.Errorf("nope"))
+	stub.WithFailure(errors.New("nope"))
 
 	_, err := process.Background(ctx, []string{"/bin/meeecho", "1"})
 	require.EqualError(t, err, "/bin/meeecho 1: nope")
@@ -51,7 +51,7 @@ func TestStubCallback(t *testing.T) {
 		if err != nil {
 			return err
 		}
-		return fmt.Errorf("yep")
+		return errors.New("yep")
 	})
 
 	_, err := process.Background(ctx, []string{"/bin/meeecho", "1"})
@@ -70,7 +70,7 @@ func TestStubResponses(t *testing.T) {
 	stub.
 		WithStdoutFor("qux 1", "first").
 		WithStdoutFor("qux 2", "second").
-		WithFailureFor("qux 3", fmt.Errorf("nope"))
+		WithFailureFor("qux 3", errors.New("nope"))
 
 	first, err := process.Background(ctx, []string{"/path/is/irrelevant/qux", "1"})
 	require.NoError(t, err)
diff --git a/libs/sync/event.go b/libs/sync/event.go
index 05821a477..510a01954 100644
--- a/libs/sync/event.go
+++ b/libs/sync/event.go
@@ -52,10 +52,10 @@ func (e *EventChanges) IsEmpty() bool {
 func (e *EventChanges) String() string {
 	var changes []string
 	if len(e.Put) > 0 {
-		changes = append(changes, fmt.Sprintf("PUT: %s", strings.Join(e.Put, ", ")))
+		changes = append(changes, "PUT: "+strings.Join(e.Put, ", "))
 	}
 	if len(e.Delete) > 0 {
-		changes = append(changes, fmt.Sprintf("DELETE: %s", strings.Join(e.Delete, ", ")))
+		changes = append(changes, "DELETE: "+strings.Join(e.Delete, ", "))
 	}
 	return strings.Join(changes, ", ")
 }
@@ -70,7 +70,7 @@ func (e *EventStart) String() string {
 		return ""
 	}
 
-	return fmt.Sprintf("Action: %s", e.EventChanges.String())
+	return "Action: " + e.EventChanges.String()
 }
 
 func newEventStart(seq int, put, delete []string) Event {
@@ -98,9 +98,9 @@ func (e *EventSyncProgress) String() string {
 
 	switch e.Action {
 	case EventActionPut:
-		return fmt.Sprintf("Uploaded %s", e.Path)
+		return "Uploaded " + e.Path
 	case EventActionDelete:
-		return fmt.Sprintf("Deleted %s", e.Path)
+		return "Deleted " + e.Path
 	default:
 		panic("invalid action")
 	}
diff --git a/libs/sync/path.go b/libs/sync/path.go
index 97a908965..87397be4b 100644
--- a/libs/sync/path.go
+++ b/libs/sync/path.go
@@ -14,7 +14,7 @@ import (
 )
 
 func repoPathForPath(me *iam.User, remotePath string) string {
-	base := path.Clean(fmt.Sprintf("/Repos/%s", me.UserName))
+	base := path.Clean("/Repos/" + me.UserName)
 	remotePath = path.Clean(remotePath)
 	for strings.HasPrefix(path.Dir(remotePath), base) && path.Dir(remotePath) != base {
 		remotePath = path.Dir(remotePath)
diff --git a/libs/sync/sync.go b/libs/sync/sync.go
index dc2c8992a..f13fa934a 100644
--- a/libs/sync/sync.go
+++ b/libs/sync/sync.go
@@ -2,6 +2,7 @@ package sync
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	stdsync "sync"
 	"time"
@@ -93,7 +94,7 @@ func New(ctx context.Context, opts SyncOptions) (*Sync, error) {
 	// specify the workspace by its resource ID. tracked in: https://databricks.atlassian.net/browse/DECO-194
 	opts.Host = opts.WorkspaceClient.Config.Host
 	if opts.Host == "" {
-		return nil, fmt.Errorf("failed to resolve host for snapshot")
+		return nil, errors.New("failed to resolve host for snapshot")
 	}
 
 	// For full sync, we start with an empty snapshot.
diff --git a/libs/tags/tag.go b/libs/tags/tag.go
index 4e9b329ca..64eab947e 100644
--- a/libs/tags/tag.go
+++ b/libs/tags/tag.go
@@ -1,6 +1,7 @@
 package tags
 
 import (
+	"errors"
 	"fmt"
 	"regexp"
 	"strings"
@@ -21,13 +22,13 @@ type tag struct {
 
 func (t *tag) ValidateKey(s string) error {
 	if len(s) == 0 {
-		return fmt.Errorf("key must not be empty")
+		return errors.New("key must not be empty")
 	}
 	if len(s) > t.keyLength {
 		return fmt.Errorf("key length %d exceeds maximum of %d", len(s), t.keyLength)
 	}
 	if strings.ContainsFunc(s, func(r rune) bool { return !unicode.Is(latin1, r) }) {
-		return fmt.Errorf("key contains non-latin1 characters")
+		return errors.New("key contains non-latin1 characters")
 	}
 	if !t.keyPattern.MatchString(s) {
 		return fmt.Errorf("key %q does not match pattern %q", s, t.keyPattern)
@@ -40,7 +41,7 @@ func (t *tag) ValidateValue(s string) error {
 		return fmt.Errorf("value length %d exceeds maximum of %d", len(s), t.valueLength)
 	}
 	if strings.ContainsFunc(s, func(r rune) bool { return !unicode.Is(latin1, r) }) {
-		return fmt.Errorf("value contains non-latin1 characters")
+		return errors.New("value contains non-latin1 characters")
 	}
 	if !t.valuePattern.MatchString(s) {
 		return fmt.Errorf("value %q does not match pattern %q", s, t.valuePattern)
diff --git a/libs/template/config.go b/libs/template/config.go
index 8e7695b91..919ba2250 100644
--- a/libs/template/config.go
+++ b/libs/template/config.go
@@ -189,7 +189,7 @@ func (c *config) promptOnce(property *jsonschema.Schema, name, defaultVal, descr
 	c.values[name], err = property.ParseString(userInput)
 	if err != nil {
 		// Show error and retry if validation fails
-		cmdio.LogString(c.ctx, fmt.Sprintf("Validation failed: %s", err.Error()))
+		cmdio.LogString(c.ctx, "Validation failed: "+err.Error())
 		return retriableError{err: err}
 	}
 
@@ -197,7 +197,7 @@ func (c *config) promptOnce(property *jsonschema.Schema, name, defaultVal, descr
 	err = c.schema.ValidateInstance(c.values)
 	if err != nil {
 		// Show error and retry if validation fails
-		cmdio.LogString(c.ctx, fmt.Sprintf("Validation failed: %s", err.Error()))
+		cmdio.LogString(c.ctx, "Validation failed: "+err.Error())
 		return retriableError{err: err}
 	}
 	return nil
diff --git a/libs/template/materialize_test.go b/libs/template/materialize_test.go
index f7cd916e3..c9331b43f 100644
--- a/libs/template/materialize_test.go
+++ b/libs/template/materialize_test.go
@@ -2,7 +2,6 @@ package template
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"testing"
 
@@ -20,5 +19,5 @@ func TestMaterializeForNonTemplateDirectory(t *testing.T) {
 
 	// Try to materialize a non-template directory.
 	err = Materialize(ctx, "", os.DirFS(tmpDir), nil)
-	assert.EqualError(t, err, fmt.Sprintf("not a bundle template: expected to find a template schema file at %s", schemaFileName))
+	assert.EqualError(t, err, "not a bundle template: expected to find a template schema file at "+schemaFileName)
 }
diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go
index 2c14009ff..70c8de12b 100644
--- a/libs/template/renderer_test.go
+++ b/libs/template/renderer_test.go
@@ -2,7 +2,6 @@ package template
 
 import (
 	"context"
-	"fmt"
 	"io/fs"
 	"os"
 	"path"
@@ -544,7 +543,7 @@ func TestRendererErrorOnConflictingFile(t *testing.T) {
 	out, err := filer.NewLocalClient(tmpDir)
 	require.NoError(t, err)
 	err = r.persistToDisk(ctx, out)
-	assert.EqualError(t, err, fmt.Sprintf("failed to initialize template, one or more files already exist: %s", "a"))
+	assert.EqualError(t, err, "failed to initialize template, one or more files already exist: "+"a")
 }
 
 func TestRendererNoErrorOnConflictingFileIfSkipped(t *testing.T) {

From e6552231eb3e9dfbf836f2e71e983b88823dbd48 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 7 Jan 2025 17:13:12 +0100
Subject: [PATCH 030/247] Use different cache keys for different jobs (#2091)

Otherwise all those jobs compete for the same key and build cache ends
up disabled for ubuntu tests.
---
 .github/workflows/push.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
index a4a35420a..db214d67f 100644
--- a/.github/workflows/push.yml
+++ b/.github/workflows/push.yml
@@ -68,6 +68,10 @@ jobs:
       - uses: actions/setup-go@v5
         with:
           go-version: 1.23.4
+          # Use different schema from regular job, to avoid overwriting the same key
+          cache-dependency-path: |
+            go.sum
+            .golangci.yaml
       - name: Run go mod tidy
         run: |
           go mod tidy
@@ -92,6 +96,10 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: 1.23.4
+          # Use different schema from regular job, to avoid overwriting the same key
+          cache-dependency-path: |
+            go.sum
+            bundle/internal/schema/*.*
 
       - name: Verify that the schema is up to date
         run: |

From 43420d01adf5af7b3ef7368e2ace78bdb7325601 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 7 Jan 2025 18:10:49 +0100
Subject: [PATCH 031/247] Run push.yml periodically with clean cache (#2092)

This ensures that our build still works with clean cache and also
populates build/test cache, speeding up test runs.
---
 .github/workflows/push.yml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
index db214d67f..b71b23c4b 100644
--- a/.github/workflows/push.yml
+++ b/.github/workflows/push.yml
@@ -13,12 +13,26 @@ on:
     # seed the build cache.
     branches:
       - main
+  schedule:
+    - cron: '0 0,12 * * *'  # Runs at 00:00 and 12:00 UTC daily
 
 env:
   GOTESTSUM_FORMAT: github-actions
 
 jobs:
+  cleanups:
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+    steps:
+      - name: Clean up cache if running on schedule
+        if: ${{ github.event_name == 'schedule' }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: gh cache delete --all --repo databricks/cli || true
+
   tests:
+    needs: cleanups
     runs-on: ${{ matrix.os }}
 
     strategy:
@@ -61,6 +75,7 @@ jobs:
         run: make test
 
   golangci:
+    needs: cleanups
     name: lint
     runs-on: ubuntu-latest
     steps:
@@ -86,6 +101,7 @@ jobs:
           args: --timeout=15m
 
   validate-bundle-schema:
+    needs: cleanups
     runs-on: ubuntu-latest
 
     steps:

From 02c7df39f6ce69234ac512d9080c64aaea3378c0 Mon Sep 17 00:00:00 2001
From: Gleb Kanterov <kanterov@users.noreply.github.com>
Date: Wed, 8 Jan 2025 10:29:45 +0100
Subject: [PATCH 032/247] Add 'experimental/python' support (#2052)

## Changes
Add `experimental/python` section replacing `experimental/pydabs`.

Add 2 new mutators into existing pipeline:
- `ApplyPythonMutator(load_resources)` - loads resources from Python
code
- `ApplyPythonMutator(apply_mutators)` - transforms existing resources
defined in Python/YAML

Example:
```yaml
experimental:
  python:
    resources:
    - "resources:load_resources"
    mutators:
    - "mutators:add_email_notifications"
```

## Tests
Unit tests and manually

---------

Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>
---
 bundle/config/experimental.go                 |  24 ++
 .../config/mutator/python/python_mutator.go   | 160 +++++++++++---
 .../mutator/python/python_mutator_test.go     | 207 ++++++++++++------
 bundle/internal/schema/annotations.yml        |  21 ++
 bundle/phases/initialize.go                   |   2 +
 bundle/schema/jsonschema.json                 |  34 +++
 6 files changed, 354 insertions(+), 94 deletions(-)

diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go
index 4c787168f..7ecac5d7d 100644
--- a/bundle/config/experimental.go
+++ b/bundle/config/experimental.go
@@ -27,9 +27,33 @@ type Experimental struct {
 	// PyDABs determines whether to load the 'databricks-pydabs' package.
 	//
 	// PyDABs allows to define bundle configuration using Python.
+	// PyDABs is deprecated use Python instead.
 	PyDABs PyDABs `json:"pydabs,omitempty"`
+
+	// Python configures loading of Python code defined with 'databricks-bundles' package.
+	Python Python `json:"python,omitempty"`
 }
 
+type Python struct {
+	// Resources contains a list of fully qualified function paths to load resources
+	// defined in Python code.
+	//
+	// Example: ["my_project.resources:load_resources"]
+	Resources []string `json:"resources"`
+
+	// Mutators contains a list of fully qualified function paths to mutator functions.
+	//
+	// Example: ["my_project.mutators:add_default_cluster"]
+	Mutators []string `json:"mutators"`
+
+	// VEnvPath is path to the virtual environment.
+	//
+	// If enabled, Python code will execute within this environment. If disabled,
+	// it defaults to using the Python interpreter available in the current shell.
+	VEnvPath string `json:"venv_path,omitempty"`
+}
+
+// PyDABs is deprecated use Python instead
 type PyDABs struct {
 	// Enabled is a flag to enable the feature.
 	Enabled bool `json:"enabled,omitempty"`
diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go
index 69c1a5dd6..8009ab243 100644
--- a/bundle/config/mutator/python/python_mutator.go
+++ b/bundle/config/mutator/python/python_mutator.go
@@ -9,6 +9,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"reflect"
 	"strings"
 
 	"github.com/databricks/databricks-sdk-go/logger"
@@ -40,6 +41,8 @@ const (
 	// We also open for possibility of appending other sections of bundle configuration,
 	// for example, adding new variables. However, this is not supported yet, and CLI rejects
 	// such changes.
+	//
+	// Deprecated, left for backward-compatibility with PyDABs.
 	PythonMutatorPhaseLoad phase = "load"
 
 	// PythonMutatorPhaseInit is the phase after bundle configuration was loaded, and
@@ -59,7 +62,46 @@ const (
 	// PyDABs can output YAML containing references to variables, and CLI should resolve them.
 	//
 	// Existing resources can't be removed, and CLI rejects such changes.
+	//
+	// Deprecated, left for backward-compatibility with PyDABs.
 	PythonMutatorPhaseInit phase = "init"
+
+	// PythonMutatorPhaseLoadResources is the phase in which YAML configuration was loaded.
+	//
+	// At this stage, we execute Python code to load resources defined in Python.
+	//
+	// During this process, Python code can access:
+	// - selected deployment target
+	// - bundle variable values
+	// - variables provided through CLI argument or environment variables
+	//
+	// The following is not available:
+	// - variables referencing other variables are in unresolved format
+	//
+	// Python code can output YAML referencing variables, and CLI should resolve them.
+	//
+	// Existing resources can't be removed or modified, and CLI rejects such changes.
+	// While it's called 'load_resources', this phase is executed in 'init' phase of mutator pipeline.
+	PythonMutatorPhaseLoadResources phase = "load_resources"
+
+	// PythonMutatorPhaseApplyMutators is the phase in which resources defined in YAML or Python
+	// are already loaded.
+	//
+	// At this stage, we execute Python code to mutate resources defined in YAML or Python.
+	//
+	// During this process, Python code can access:
+	// - selected deployment target
+	// - bundle variable values
+	// - variables provided through CLI argument or environment variables
+	//
+	// The following is not available:
+	// - variables referencing other variables are in unresolved format
+	//
+	// Python code can output YAML referencing variables, and CLI should resolve them.
+	//
+	// Resources can't be added or removed, and CLI rejects such changes. Python code is
+	// allowed to modify existing resources, but not other parts of bundle configuration.
+	PythonMutatorPhaseApplyMutators phase = "apply_mutators"
 )
 
 type pythonMutator struct {
@@ -76,18 +118,64 @@ func (m *pythonMutator) Name() string {
 	return fmt.Sprintf("PythonMutator(%s)", m.phase)
 }
 
-func getExperimental(b *bundle.Bundle) config.Experimental {
-	if b.Config.Experimental == nil {
-		return config.Experimental{}
+// opts is a common structure for deprecated PyDABs and upcoming Python
+// configuration sections
+type opts struct {
+	enabled bool
+
+	venvPath string
+}
+
+// getOpts adapts deprecated PyDABs and upcoming Python configuration
+// into a common structure.
+func getOpts(b *bundle.Bundle, phase phase) (opts, error) {
+	experimental := b.Config.Experimental
+	if experimental == nil {
+		return opts{}, nil
 	}
 
-	return *b.Config.Experimental
+	// using reflect.DeepEquals in case we add more fields
+	pydabsEnabled := !reflect.DeepEqual(experimental.PyDABs, config.PyDABs{})
+	pythonEnabled := !reflect.DeepEqual(experimental.Python, config.Python{})
+
+	if pydabsEnabled && pythonEnabled {
+		return opts{}, errors.New("both experimental/pydabs and experimental/python are enabled, only one can be enabled")
+	} else if pydabsEnabled {
+		if !experimental.PyDABs.Enabled {
+			return opts{}, nil
+		}
+
+		// don't execute for phases for 'python' section
+		if phase == PythonMutatorPhaseInit || phase == PythonMutatorPhaseLoad {
+			return opts{
+				enabled:  true,
+				venvPath: experimental.PyDABs.VEnvPath,
+			}, nil
+		} else {
+			return opts{}, nil
+		}
+	} else if pythonEnabled {
+		// don't execute for phases for 'pydabs' section
+		if phase == PythonMutatorPhaseLoadResources || phase == PythonMutatorPhaseApplyMutators {
+			return opts{
+				enabled:  true,
+				venvPath: experimental.Python.VEnvPath,
+			}, nil
+		} else {
+			return opts{}, nil
+		}
+	} else {
+		return opts{}, nil
+	}
 }
 
 func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-	experimental := getExperimental(b)
+	opts, err := getOpts(b, m.phase)
+	if err != nil {
+		return diag.Errorf("failed to apply python mutator: %s", err)
+	}
 
-	if !experimental.PyDABs.Enabled {
+	if !opts.enabled {
 		return nil
 	}
 
@@ -95,8 +183,8 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
 	var mutateDiags diag.Diagnostics
 	mutateDiagsHasError := errors.New("unexpected error")
 
-	err := b.Config.Mutate(func(leftRoot dyn.Value) (dyn.Value, error) {
-		pythonPath, err := detectExecutable(ctx, experimental.PyDABs.VEnvPath)
+	err = b.Config.Mutate(func(leftRoot dyn.Value) (dyn.Value, error) {
+		pythonPath, err := detectExecutable(ctx, opts.venvPath)
 		if err != nil {
 			return dyn.InvalidValue, fmt.Errorf("failed to get Python interpreter path: %w", err)
 		}
@@ -137,7 +225,7 @@ func createCacheDir(ctx context.Context) (string, error) {
 	// support the same env variable as in b.CacheDir
 	if tempDir, exists := env.TempDir(ctx); exists {
 		// use 'default' as target name
-		cacheDir := filepath.Join(tempDir, "default", "pydabs")
+		cacheDir := filepath.Join(tempDir, "default", "python")
 
 		err := os.MkdirAll(cacheDir, 0o700)
 		if err != nil {
@@ -147,7 +235,7 @@ func createCacheDir(ctx context.Context) (string, error) {
 		return cacheDir, nil
 	}
 
-	return os.MkdirTemp("", "-pydabs")
+	return os.MkdirTemp("", "-python")
 }
 
 func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir, rootPath, pythonPath string, root dyn.Value) (dyn.Value, diag.Diagnostics) {
@@ -203,7 +291,7 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir, rootPath
 	}
 
 	// process can fail without reporting errors in diagnostics file or creating it, for instance,
-	// venv doesn't have PyDABs library installed
+	// venv doesn't have 'databricks-bundles' library installed
 	if processErr != nil {
 		diagnostic := diag.Diagnostic{
 			Severity: diag.Error,
@@ -226,16 +314,15 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir, rootPath
 	return output, pythonDiagnostics
 }
 
-const installExplanation = `If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies,
-and that the wheel is installed in the Python environment:
+const pythonInstallExplanation = `Ensure that 'databricks-bundles' is installed in Python environment:
 
-  $ .venv/bin/pip install -e .
+  $ .venv/bin/pip install databricks-bundles
 
 If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml,
 or activate the environment before running CLI commands:
 
   experimental:
-    pydabs:
+    python:
       venv_path: .venv
 `
 
@@ -245,9 +332,9 @@ or activate the environment before running CLI commands:
 func explainProcessErr(stderr string) string {
 	// implemented in cpython/Lib/runpy.py and portable across Python 3.x, including pypy
 	if strings.Contains(stderr, "Error while finding module specification for 'databricks.bundles.build'") {
-		summary := color.CyanString("Explanation: ") + "'databricks-pydabs' library is not installed in the Python environment.\n"
+		summary := color.CyanString("Explanation: ") + "'databricks-bundles' library is not installed in the Python environment.\n"
 
-		return stderr + "\n" + summary + "\n" + installExplanation
+		return stderr + "\n" + summary + "\n" + pythonInstallExplanation
 	}
 
 	return stderr
@@ -277,10 +364,10 @@ func loadOutputFile(rootPath, outputPath string) (dyn.Value, diag.Diagnostics) {
 	//
 	// virtualPath has to stay in rootPath, because locations outside root path are not allowed:
 	//
-	//   Error: path /var/folders/.../pydabs/dist/*.whl is not contained in bundle root path
+	//   Error: path /var/folders/.../python/dist/*.whl is not contained in bundle root path
 	//
 	// for that, we pass virtualPath instead of outputPath as file location
-	virtualPath, err := filepath.Abs(filepath.Join(rootPath, "__generated_by_pydabs__.yml"))
+	virtualPath, err := filepath.Abs(filepath.Join(rootPath, "__generated_by_python__.yml"))
 	if err != nil {
 		return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to get absolute path: %w", err))
 	}
@@ -334,19 +421,23 @@ func loadDiagnosticsFile(path string) (diag.Diagnostics, error) {
 func createOverrideVisitor(ctx context.Context, phase phase) (merge.OverrideVisitor, error) {
 	switch phase {
 	case PythonMutatorPhaseLoad:
-		return createLoadOverrideVisitor(ctx), nil
+		return createLoadResourcesOverrideVisitor(ctx), nil
 	case PythonMutatorPhaseInit:
-		return createInitOverrideVisitor(ctx), nil
+		return createInitOverrideVisitor(ctx, insertResourceModeAllow), nil
+	case PythonMutatorPhaseLoadResources:
+		return createLoadResourcesOverrideVisitor(ctx), nil
+	case PythonMutatorPhaseApplyMutators:
+		return createInitOverrideVisitor(ctx, insertResourceModeDisallow), nil
 	default:
 		return merge.OverrideVisitor{}, fmt.Errorf("unknown phase: %s", phase)
 	}
 }
 
-// createLoadOverrideVisitor creates an override visitor for the load phase.
+// createLoadResourcesOverrideVisitor creates an override visitor for the load_resources phase.
 //
-// During load, it's only possible to create new resources, and not modify or
+// During load_resources, it's only possible to create new resources, and not modify or
 // delete existing ones.
-func createLoadOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
+func createLoadResourcesOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
 	resourcesPath := dyn.NewPath(dyn.Key("resources"))
 	jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"))
 
@@ -385,11 +476,21 @@ func createLoadOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
 	}
 }
 
+// insertResourceMode controls whether createInitOverrideVisitor allows or disallows inserting new resources.
+type insertResourceMode int
+
+const (
+	insertResourceModeDisallow insertResourceMode = iota
+	insertResourceModeAllow    insertResourceMode = iota
+)
+
 // createInitOverrideVisitor creates an override visitor for the init phase.
 //
 // During the init phase it's possible to create new resources, modify existing
 // resources, but not delete existing resources.
-func createInitOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
+//
+// If mode is insertResourceModeDisallow, it matching expected behaviour of apply_mutators
+func createInitOverrideVisitor(ctx context.Context, mode insertResourceMode) merge.OverrideVisitor {
 	resourcesPath := dyn.NewPath(dyn.Key("resources"))
 	jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"))
 
@@ -424,6 +525,11 @@ func createInitOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
 				return dyn.InvalidValue, fmt.Errorf("unexpected change at %q (insert)", valuePath.String())
 			}
 
+			insertResource := len(valuePath) == len(jobsPath)+1
+			if mode == insertResourceModeDisallow && insertResource {
+				return dyn.InvalidValue, fmt.Errorf("unexpected change at %q (insert)", valuePath.String())
+			}
+
 			log.Debugf(ctx, "Insert value at %q", valuePath.String())
 
 			return right, nil
@@ -441,9 +547,9 @@ func createInitOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
 }
 
 func isOmitemptyDelete(left dyn.Value) bool {
-	// PyDABs can omit empty sequences/mappings in output, because we don't track them as optional,
+	// Python output can omit empty sequences/mappings, because we don't track them as optional,
 	// there is no semantic difference between empty and missing, so we keep them as they were before
-	// PyDABs deleted them.
+	// Python mutator deleted them.
 
 	switch left.Kind() {
 	case dyn.KindMap:
diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go
index ff21f8ed9..d51572c8a 100644
--- a/bundle/config/mutator/python/python_mutator_test.go
+++ b/bundle/config/mutator/python/python_mutator_test.go
@@ -40,13 +40,25 @@ func TestPythonMutator_Name_init(t *testing.T) {
 	assert.Equal(t, "PythonMutator(init)", mutator.Name())
 }
 
-func TestPythonMutator_load(t *testing.T) {
+func TestPythonMutator_Name_loadResources(t *testing.T) {
+	mutator := PythonMutator(PythonMutatorPhaseLoadResources)
+
+	assert.Equal(t, "PythonMutator(load_resources)", mutator.Name())
+}
+
+func TestPythonMutator_Name_applyMutators(t *testing.T) {
+	mutator := PythonMutator(PythonMutatorPhaseApplyMutators)
+
+	assert.Equal(t, "PythonMutator(apply_mutators)", mutator.Name())
+}
+
+func TestPythonMutator_loadResources(t *testing.T) {
 	withFakeVEnv(t, ".venv")
 
 	b := loadYaml("databricks.yml", `
       experimental:
-        pydabs:
-          enabled: true
+        python:
+          resources: ["resources:load_resources"]
           venv_path: .venv
       resources:
         jobs:
@@ -60,12 +72,12 @@ func TestPythonMutator_load(t *testing.T) {
 			"-m",
 			"databricks.bundles.build",
 			"--phase",
-			"load",
+			"load_resources",
 		},
 		`{
 			"experimental": {
-				"pydabs": {
-					"enabled": true,
+				"python": {
+					"resources": ["resources:load_resources"],
 					"venv_path": ".venv"
 				}
 			},
@@ -83,7 +95,7 @@ func TestPythonMutator_load(t *testing.T) {
 		`{"severity": "warning", "summary": "job doesn't have any tasks", "location": {"file": "src/examples/file.py", "line": 10, "column": 5}}`,
 	)
 
-	mutator := PythonMutator(PythonMutatorPhaseLoad)
+	mutator := PythonMutator(PythonMutatorPhaseLoadResources)
 	diags := bundle.Apply(ctx, b, mutator)
 
 	assert.NoError(t, diags.Error())
@@ -109,13 +121,12 @@ func TestPythonMutator_load(t *testing.T) {
 	}, diags[0].Locations)
 }
 
-func TestPythonMutator_load_disallowed(t *testing.T) {
+func TestPythonMutator_loadResources_disallowed(t *testing.T) {
 	withFakeVEnv(t, ".venv")
-
 	b := loadYaml("databricks.yml", `
       experimental:
-        pydabs:
-          enabled: true
+        python:
+          resources: ["resources:load_resources"]
           venv_path: .venv
       resources:
         jobs:
@@ -129,12 +140,12 @@ func TestPythonMutator_load_disallowed(t *testing.T) {
 			"-m",
 			"databricks.bundles.build",
 			"--phase",
-			"load",
+			"load_resources",
 		},
 		`{
 			"experimental": {
-				"pydabs": {
-					"enabled": true,
+				"python": {
+					"resources": ["resources:load_resources"],
 					"venv_path": ".venv"
 				}
 			},
@@ -148,20 +159,20 @@ func TestPythonMutator_load_disallowed(t *testing.T) {
 			}
 		}`, "")
 
-	mutator := PythonMutator(PythonMutatorPhaseLoad)
+	mutator := PythonMutator(PythonMutatorPhaseLoadResources)
 	diag := bundle.Apply(ctx, b, mutator)
 
 	assert.EqualError(t, diag.Error(), "unexpected change at \"resources.jobs.job0.description\" (insert)")
 }
 
-func TestPythonMutator_init(t *testing.T) {
+func TestPythonMutator_applyMutators(t *testing.T) {
 	withFakeVEnv(t, ".venv")
-
 	b := loadYaml("databricks.yml", `
       experimental:
-        pydabs:
-          enabled: true
+        python:
           venv_path: .venv
+          mutators:
+            - "mutators:add_description"
       resources:
         jobs:
           job0:
@@ -174,13 +185,13 @@ func TestPythonMutator_init(t *testing.T) {
 			"-m",
 			"databricks.bundles.build",
 			"--phase",
-			"init",
+			"apply_mutators",
 		},
 		`{
 			"experimental": {
-				"pydabs": {
-					"enabled": true,
-					"venv_path": ".venv"
+				"python": {
+					"venv_path": ".venv",
+					"mutators": ["mutators:add_description"]
 				}
 			},
 			"resources": {
@@ -193,7 +204,7 @@ func TestPythonMutator_init(t *testing.T) {
 			}
 		}`, "")
 
-	mutator := PythonMutator(PythonMutatorPhaseInit)
+	mutator := PythonMutator(PythonMutatorPhaseApplyMutators)
 	diag := bundle.Apply(ctx, b, mutator)
 
 	assert.NoError(t, diag.Error())
@@ -208,12 +219,12 @@ func TestPythonMutator_init(t *testing.T) {
 		require.NoError(t, err)
 		assert.Equal(t, "databricks.yml", name.Location().File)
 
-		// 'description' was updated by PyDABs and has location of generated file until
+		// 'description' was updated by Python code and has location of generated file until
 		// we implement source maps
 		description, err := dyn.GetByPath(v, dyn.MustPathFromString("resources.jobs.job0.description"))
 		require.NoError(t, err)
 
-		expectedVirtualPath, err := filepath.Abs("__generated_by_pydabs__.yml")
+		expectedVirtualPath, err := filepath.Abs("__generated_by_python__.yml")
 		require.NoError(t, err)
 		assert.Equal(t, expectedVirtualPath, description.Location().File)
 
@@ -224,12 +235,12 @@ func TestPythonMutator_init(t *testing.T) {
 
 func TestPythonMutator_badOutput(t *testing.T) {
 	withFakeVEnv(t, ".venv")
-
 	b := loadYaml("databricks.yml", `
       experimental:
-        pydabs:
-          enabled: true
+        python:
           venv_path: .venv
+          resources:
+            - "resources:load_resources"
       resources:
         jobs:
           job0:
@@ -242,7 +253,7 @@ func TestPythonMutator_badOutput(t *testing.T) {
 			"-m",
 			"databricks.bundles.build",
 			"--phase",
-			"load",
+			"load_resources",
 		},
 		`{
 			"resources": {
@@ -254,7 +265,7 @@ func TestPythonMutator_badOutput(t *testing.T) {
 			}
 		}`, "")
 
-	mutator := PythonMutator(PythonMutatorPhaseLoad)
+	mutator := PythonMutator(PythonMutatorPhaseLoadResources)
 	diag := bundle.Apply(ctx, b, mutator)
 
 	assert.EqualError(t, diag.Error(), "unknown field: unknown_property")
@@ -270,34 +281,63 @@ func TestPythonMutator_disabled(t *testing.T) {
 	assert.NoError(t, diag.Error())
 }
 
-func TestPythonMutator_venvRequired(t *testing.T) {
-	b := loadYaml("databricks.yml", `
-      experimental:
-        pydabs:
-          enabled: true`)
-
-	ctx := context.Background()
-	mutator := PythonMutator(PythonMutatorPhaseLoad)
-	diag := bundle.Apply(ctx, b, mutator)
-
-	assert.Error(t, diag.Error(), "\"experimental.enable_pydabs\" is enabled, but \"experimental.venv.path\" is not set")
-}
-
 func TestPythonMutator_venvNotFound(t *testing.T) {
 	expectedError := fmt.Sprintf("failed to get Python interpreter path: can't find %q, check if virtualenv is created", interpreterPath("bad_path"))
 
 	b := loadYaml("databricks.yml", `
       experimental:
-        pydabs:
-          enabled: true
-          venv_path: bad_path`)
+        python:
+          venv_path: bad_path
+          resources:
+            - "resources:load_resources"`)
 
-	mutator := PythonMutator(PythonMutatorPhaseInit)
+	mutator := PythonMutator(PythonMutatorPhaseLoadResources)
 	diag := bundle.Apply(context.Background(), b, mutator)
 
 	assert.EqualError(t, diag.Error(), expectedError)
 }
 
+func TestGetOps_Python(t *testing.T) {
+	actual, err := getOpts(&bundle.Bundle{
+		Config: config.Root{
+			Experimental: &config.Experimental{
+				Python: config.Python{
+					VEnvPath: ".venv",
+					Resources: []string{
+						"resources:load_resources",
+					},
+				},
+			},
+		},
+	}, PythonMutatorPhaseLoadResources)
+
+	assert.NoError(t, err)
+	assert.Equal(t, opts{venvPath: ".venv", enabled: true}, actual)
+}
+
+func TestGetOps_PyDABs(t *testing.T) {
+	actual, err := getOpts(&bundle.Bundle{
+		Config: config.Root{
+			Experimental: &config.Experimental{
+				PyDABs: config.PyDABs{
+					VEnvPath: ".venv",
+					Enabled:  true,
+				},
+			},
+		},
+	}, PythonMutatorPhaseInit)
+
+	assert.NoError(t, err)
+	assert.Equal(t, opts{venvPath: ".venv", enabled: true}, actual)
+}
+
+func TestGetOps_empty(t *testing.T) {
+	actual, err := getOpts(&bundle.Bundle{}, PythonMutatorPhaseLoadResources)
+
+	assert.NoError(t, err)
+	assert.Equal(t, opts{enabled: false}, actual)
+}
+
 type createOverrideVisitorTestCase struct {
 	name        string
 	updatePath  dyn.Path
@@ -315,8 +355,8 @@ func TestCreateOverrideVisitor(t *testing.T) {
 
 	testCases := []createOverrideVisitorTestCase{
 		{
-			name:        "load: can't change an existing job",
-			phase:       PythonMutatorPhaseLoad,
+			name:        "load_resources: can't change an existing job",
+			phase:       PythonMutatorPhaseLoadResources,
 			updatePath:  dyn.MustPathFromString("resources.jobs.job0.name"),
 			deletePath:  dyn.MustPathFromString("resources.jobs.job0.name"),
 			insertPath:  dyn.MustPathFromString("resources.jobs.job0.name"),
@@ -325,32 +365,32 @@ func TestCreateOverrideVisitor(t *testing.T) {
 			updateError: errors.New("unexpected change at \"resources.jobs.job0.name\" (update)"),
 		},
 		{
-			name:        "load: can't delete an existing job",
-			phase:       PythonMutatorPhaseLoad,
+			name:        "load_resources: can't delete an existing job",
+			phase:       PythonMutatorPhaseLoadResources,
 			deletePath:  dyn.MustPathFromString("resources.jobs.job0"),
 			deleteError: errors.New("unexpected change at \"resources.jobs.job0\" (delete)"),
 		},
 		{
-			name:        "load: can insert 'resources'",
-			phase:       PythonMutatorPhaseLoad,
+			name:        "load_resources: can insert 'resources'",
+			phase:       PythonMutatorPhaseLoadResources,
 			insertPath:  dyn.MustPathFromString("resources"),
 			insertError: nil,
 		},
 		{
-			name:        "load: can insert 'resources.jobs'",
-			phase:       PythonMutatorPhaseLoad,
+			name:        "load_resources: can insert 'resources.jobs'",
+			phase:       PythonMutatorPhaseLoadResources,
 			insertPath:  dyn.MustPathFromString("resources.jobs"),
 			insertError: nil,
 		},
 		{
-			name:        "load: can insert a job",
-			phase:       PythonMutatorPhaseLoad,
+			name:        "load_resources: can insert a job",
+			phase:       PythonMutatorPhaseLoadResources,
 			insertPath:  dyn.MustPathFromString("resources.jobs.job0"),
 			insertError: nil,
 		},
 		{
-			name:        "load: can't change include",
-			phase:       PythonMutatorPhaseLoad,
+			name:        "load_resources: can't change include",
+			phase:       PythonMutatorPhaseLoadResources,
 			deletePath:  dyn.MustPathFromString("include[0]"),
 			insertPath:  dyn.MustPathFromString("include[0]"),
 			updatePath:  dyn.MustPathFromString("include[0]"),
@@ -402,6 +442,40 @@ func TestCreateOverrideVisitor(t *testing.T) {
 			insertError: errors.New("unexpected change at \"include[0]\" (insert)"),
 			updateError: errors.New("unexpected change at \"include[0]\" (update)"),
 		},
+		{
+			name:        "apply_mutators: can't delete an existing job",
+			phase:       PythonMutatorPhaseInit,
+			deletePath:  dyn.MustPathFromString("resources.jobs.job0"),
+			deleteError: errors.New("unexpected change at \"resources.jobs.job0\" (delete)"),
+		},
+		{
+			name:        "apply_mutators: can insert 'resources'",
+			phase:       PythonMutatorPhaseApplyMutators,
+			insertPath:  dyn.MustPathFromString("resources"),
+			insertError: nil,
+		},
+		{
+			name:        "apply_mutators: can insert 'resources.jobs'",
+			phase:       PythonMutatorPhaseApplyMutators,
+			insertPath:  dyn.MustPathFromString("resources.jobs"),
+			insertError: nil,
+		},
+		{
+			name:        "apply_mutators: can't insert a job",
+			phase:       PythonMutatorPhaseApplyMutators,
+			insertPath:  dyn.MustPathFromString("resources.jobs.job0"),
+			insertError: errors.New("unexpected change at \"resources.jobs.job0\" (insert)"),
+		},
+		{
+			name:        "apply_mutators: can't change include",
+			phase:       PythonMutatorPhaseApplyMutators,
+			deletePath:  dyn.MustPathFromString("include[0]"),
+			insertPath:  dyn.MustPathFromString("include[0]"),
+			updatePath:  dyn.MustPathFromString("include[0]"),
+			deleteError: errors.New("unexpected change at \"include[0]\" (delete)"),
+			insertError: errors.New("unexpected change at \"include[0]\" (insert)"),
+			updateError: errors.New("unexpected change at \"include[0]\" (update)"),
+		},
 	}
 
 	for _, tc := range testCases {
@@ -459,9 +533,9 @@ type overrideVisitorOmitemptyTestCase struct {
 }
 
 func TestCreateOverrideVisitor_omitempty(t *testing.T) {
-	// PyDABs can omit empty sequences/mappings in output, because we don't track them as optional,
+	// Python output can omit empty sequences/mappings in output, because we don't track them as optional,
 	// there is no semantic difference between empty and missing, so we keep them as they were before
-	// PyDABs deleted them.
+	// Python code deleted them.
 
 	allPhases := []phase{PythonMutatorPhaseLoad, PythonMutatorPhaseInit}
 	location := dyn.Location{
@@ -568,18 +642,17 @@ func TestExplainProcessErr(t *testing.T) {
 	stderr := "/home/test/.venv/bin/python3: Error while finding module specification for 'databricks.bundles.build' (ModuleNotFoundError: No module named 'databricks')\n"
 	expected := `/home/test/.venv/bin/python3: Error while finding module specification for 'databricks.bundles.build' (ModuleNotFoundError: No module named 'databricks')
 
-Explanation: 'databricks-pydabs' library is not installed in the Python environment.
+Explanation: 'databricks-bundles' library is not installed in the Python environment.
 
-If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies,
-and that the wheel is installed in the Python environment:
+Ensure that 'databricks-bundles' is installed in Python environment:
 
-  $ .venv/bin/pip install -e .
+  $ .venv/bin/pip install databricks-bundles
 
 If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml,
 or activate the environment before running CLI commands:
 
   experimental:
-    pydabs:
+    python:
       venv_path: .venv
 `
 
diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml
index 84f6753e3..5283a431b 100644
--- a/bundle/internal/schema/annotations.yml
+++ b/bundle/internal/schema/annotations.yml
@@ -69,6 +69,9 @@ github.com/databricks/cli/bundle/config.Experimental:
   "pydabs":
     "description": |-
       The PyDABs configuration.
+  "python":
+    "description": |-
+      Configures loading of Python code defined with 'databricks-bundles' package.
   "python_wheel_wrapper":
     "description": |-
       Whether to use a Python wheel wrapper
@@ -125,6 +128,24 @@ github.com/databricks/cli/bundle/config.PyDABs:
   "venv_path":
     "description": |-
       The Python virtual environment path
+github.com/databricks/cli/bundle/config.Python:
+  "mutators":
+    "description": |-
+      Mutators contains a list of fully qualified function paths to mutator functions.
+
+      Example: ["my_project.mutators:add_default_cluster"]
+  "resources":
+    "description": |-
+      Resources contains a list of fully qualified function paths to load resources
+      defined in Python code.
+
+      Example: ["my_project.resources:load_resources"]
+  "venv_path":
+    "description": |-
+      VEnvPath is path to the virtual environment.
+
+      If enabled, Python code will execute within this environment. If disabled,
+      it defaults to using the Python interpreter available in the current shell.
 github.com/databricks/cli/bundle/config.Resources:
   "clusters":
     "description": |-
diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go
index 6fa0e5fed..f0cbc00c2 100644
--- a/bundle/phases/initialize.go
+++ b/bundle/phases/initialize.go
@@ -55,6 +55,8 @@ func Initialize() bundle.Mutator {
 			// ResolveVariableReferencesInComplexVariables and ResolveVariableReferences.
 			// See what is expected in PythonMutatorPhaseInit doc
 			pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseInit),
+			pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseLoadResources),
+			pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseApplyMutators),
 			mutator.ResolveVariableReferencesInLookup(),
 			mutator.ResolveResourceReferences(),
 			mutator.ResolveVariableReferencesInComplexVariables(),
diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json
index 9a352ebb2..2f78ffcca 100644
--- a/bundle/schema/jsonschema.json
+++ b/bundle/schema/jsonschema.json
@@ -1100,6 +1100,10 @@
                       "description": "The PyDABs configuration.",
                       "$ref": "#/$defs/github.com/databricks/cli/bundle/config.PyDABs"
                     },
+                    "python": {
+                      "description": "Configures loading of Python code defined with 'databricks-bundles' package.",
+                      "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Python"
+                    },
                     "python_wheel_wrapper": {
                       "description": "Whether to use a Python wheel wrapper",
                       "$ref": "#/$defs/bool"
@@ -1234,6 +1238,36 @@
                 }
               ]
             },
+            "config.Python": {
+              "oneOf": [
+                {
+                  "type": "object",
+                  "properties": {
+                    "mutators": {
+                      "description": "Mutators contains a list of fully qualified function paths to mutator functions.\n\nExample: [\"my_project.mutators:add_default_cluster\"]",
+                      "$ref": "#/$defs/slice/string"
+                    },
+                    "resources": {
+                      "description": "Resources contains a list of fully qualified function paths to load resources\ndefined in Python code.\n\nExample: [\"my_project.resources:load_resources\"]",
+                      "$ref": "#/$defs/slice/string"
+                    },
+                    "venv_path": {
+                      "description": "VEnvPath is path to the virtual environment.\n\nIf enabled, Python code will execute within this environment. If disabled,\nit defaults to using the Python interpreter available in the current shell.",
+                      "$ref": "#/$defs/string"
+                    }
+                  },
+                  "additionalProperties": false,
+                  "required": [
+                    "resources",
+                    "mutators"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
+              ]
+            },
             "config.Resources": {
               "oneOf": [
                 {

From 8fd793b605f3fe4f2b25f76dfd6afcc6ae7985a3 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 8 Jan 2025 12:59:22 +0100
Subject: [PATCH 033/247] Clean up TestMain from integration tests to fix
 caching (#2090)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Changes
- Remove TestMain from integration tests and related checks.
- This fixes "go test" caching for integration tests.

The test_main.go files were added in
https://github.com/databricks/cli/pull/2009 to make sure integration
tests are not run as part of go test ./.... We recommend running make
test to run tests, which includes the packages to test (and excludes
integration).

## Tests
To test that caching works I ran a test twice:

```
+ CLOUD_ENV=aws
+ go test --timeout 3h -v -run TestDefaultPython/3.9 ./integration/bundle/
…
PASS
ok      github.com/databricks/cli/integration/bundle    (cached)
```
---
 integration/assumptions/main_test.go          |  13 --
 integration/bundle/main_test.go               |  13 --
 integration/cmd/alerts/main_test.go           |  13 --
 integration/cmd/api/main_test.go              |  13 --
 integration/cmd/auth/main_test.go             |  13 --
 integration/cmd/clusters/main_test.go         |  13 --
 integration/cmd/fs/main_test.go               |  13 --
 integration/cmd/jobs/main_test.go             |  13 --
 integration/cmd/main_test.go                  |  13 --
 integration/cmd/repos/main_test.go            |  13 --
 integration/cmd/secrets/main_test.go          |  13 --
 .../cmd/storage_credentials/main_test.go      |  13 --
 integration/cmd/sync/main_test.go             |  13 --
 integration/cmd/version/main_test.go          |  13 --
 integration/cmd/workspace/main_test.go        |  13 --
 integration/enforce_convention_test.go        | 116 ------------------
 integration/internal/main.go                  |  22 ----
 integration/libs/filer/main_test.go           |  13 --
 integration/libs/git/main_test.go             |  13 --
 integration/libs/locker/main_test.go          |  13 --
 integration/libs/tags/main_test.go            |  13 --
 integration/python/main_test.go               |  13 --
 22 files changed, 398 deletions(-)
 delete mode 100644 integration/assumptions/main_test.go
 delete mode 100644 integration/bundle/main_test.go
 delete mode 100644 integration/cmd/alerts/main_test.go
 delete mode 100644 integration/cmd/api/main_test.go
 delete mode 100644 integration/cmd/auth/main_test.go
 delete mode 100644 integration/cmd/clusters/main_test.go
 delete mode 100644 integration/cmd/fs/main_test.go
 delete mode 100644 integration/cmd/jobs/main_test.go
 delete mode 100644 integration/cmd/main_test.go
 delete mode 100644 integration/cmd/repos/main_test.go
 delete mode 100644 integration/cmd/secrets/main_test.go
 delete mode 100644 integration/cmd/storage_credentials/main_test.go
 delete mode 100644 integration/cmd/sync/main_test.go
 delete mode 100644 integration/cmd/version/main_test.go
 delete mode 100644 integration/cmd/workspace/main_test.go
 delete mode 100644 integration/enforce_convention_test.go
 delete mode 100644 integration/internal/main.go
 delete mode 100644 integration/libs/filer/main_test.go
 delete mode 100644 integration/libs/git/main_test.go
 delete mode 100644 integration/libs/locker/main_test.go
 delete mode 100644 integration/libs/tags/main_test.go
 delete mode 100644 integration/python/main_test.go

diff --git a/integration/assumptions/main_test.go b/integration/assumptions/main_test.go
deleted file mode 100644
index be2761385..000000000
--- a/integration/assumptions/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package assumptions_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
diff --git a/integration/bundle/main_test.go b/integration/bundle/main_test.go
deleted file mode 100644
index 1c44d0aaf..000000000
--- a/integration/bundle/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package bundle_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
diff --git a/integration/cmd/alerts/main_test.go b/integration/cmd/alerts/main_test.go
deleted file mode 100644
index 6987ade02..000000000
--- a/integration/cmd/alerts/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package alerts_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
diff --git a/integration/cmd/api/main_test.go b/integration/cmd/api/main_test.go
deleted file mode 100644
index 70d021790..000000000
--- a/integration/cmd/api/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package api_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
diff --git a/integration/cmd/auth/main_test.go b/integration/cmd/auth/main_test.go
deleted file mode 100644
index 97b1d740b..000000000
--- a/integration/cmd/auth/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package auth_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
diff --git a/integration/cmd/clusters/main_test.go b/integration/cmd/clusters/main_test.go
deleted file mode 100644
index ccd5660e7..000000000
--- a/integration/cmd/clusters/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package clusters_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
diff --git a/integration/cmd/fs/main_test.go b/integration/cmd/fs/main_test.go
deleted file mode 100644
index b9402f0b2..000000000
--- a/integration/cmd/fs/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package fs_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
diff --git a/integration/cmd/jobs/main_test.go b/integration/cmd/jobs/main_test.go
deleted file mode 100644
index 46369a526..000000000
--- a/integration/cmd/jobs/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package jobs_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
diff --git a/integration/cmd/main_test.go b/integration/cmd/main_test.go
deleted file mode 100644
index a1a5586b6..000000000
--- a/integration/cmd/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package cmd_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
diff --git a/integration/cmd/repos/main_test.go b/integration/cmd/repos/main_test.go
deleted file mode 100644
index 7eaa174bc..000000000
--- a/integration/cmd/repos/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package repos_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
diff --git a/integration/cmd/secrets/main_test.go b/integration/cmd/secrets/main_test.go
deleted file mode 100644
index a44d30671..000000000
--- a/integration/cmd/secrets/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package secrets_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
diff --git a/integration/cmd/storage_credentials/main_test.go b/integration/cmd/storage_credentials/main_test.go
deleted file mode 100644
index 14d00d966..000000000
--- a/integration/cmd/storage_credentials/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package storage_credentials_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
diff --git a/integration/cmd/sync/main_test.go b/integration/cmd/sync/main_test.go
deleted file mode 100644
index 8d9f3ca25..000000000
--- a/integration/cmd/sync/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package sync_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
diff --git a/integration/cmd/version/main_test.go b/integration/cmd/version/main_test.go
deleted file mode 100644
index 4aa5e046a..000000000
--- a/integration/cmd/version/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package version_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
diff --git a/integration/cmd/workspace/main_test.go b/integration/cmd/workspace/main_test.go
deleted file mode 100644
index 40d140eac..000000000
--- a/integration/cmd/workspace/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package workspace_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
diff --git a/integration/enforce_convention_test.go b/integration/enforce_convention_test.go
deleted file mode 100644
index cc822a6a3..000000000
--- a/integration/enforce_convention_test.go
+++ /dev/null
@@ -1,116 +0,0 @@
-package integration
-
-import (
-	"go/parser"
-	"go/token"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-	"text/template"
-
-	"golang.org/x/exp/maps"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-type packageInfo struct {
-	Name  string
-	Files []string
-}
-
-func enumeratePackages(t *testing.T) map[string]packageInfo {
-	pkgmap := make(map[string]packageInfo)
-	err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		// Skip files.
-		if !info.IsDir() {
-			return nil
-		}
-
-		// Skip the root directory and the "internal" directory.
-		if path == "." || strings.HasPrefix(path, "internal") {
-			return nil
-		}
-
-		fset := token.NewFileSet()
-		pkgs, err := parser.ParseDir(fset, path, nil, parser.ParseComments)
-		require.NoError(t, err)
-		if len(pkgs) == 0 {
-			return nil
-		}
-
-		// Expect one package per directory.
-		require.Len(t, pkgs, 1, "Directory %s contains more than one package", path)
-		v := maps.Values(pkgs)[0]
-
-		// Record the package.
-		pkgmap[path] = packageInfo{
-			Name:  v.Name,
-			Files: maps.Keys(v.Files),
-		}
-		return nil
-	})
-	require.NoError(t, err)
-	return pkgmap
-}
-
-// TestEnforcePackageNames checks that all integration test package names use the "_test" suffix.
-// We enforce this package name to avoid package name aliasing.
-func TestEnforcePackageNames(t *testing.T) {
-	pkgmap := enumeratePackages(t)
-	for _, pkg := range pkgmap {
-		assert.True(t, strings.HasSuffix(pkg.Name, "_test"), "Package name %s does not end with _test", pkg.Name)
-	}
-}
-
-var mainTestTemplate = template.Must(template.New("main_test").Parse(
-	`package {{.Name}}
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
-`))
-
-func TestEnforceMainTest(t *testing.T) {
-	pkgmap := enumeratePackages(t)
-	for dir, pkg := range pkgmap {
-		found := false
-		for _, file := range pkg.Files {
-			if filepath.Base(file) == "main_test.go" {
-				found = true
-				break
-			}
-		}
-
-		// Expect a "main_test.go" file in each package.
-		assert.True(t, found, "Directory %s does not contain a main_test.go file", dir)
-	}
-}
-
-func TestWriteMainTest(t *testing.T) {
-	t.Skip("Uncomment to write main_test.go files")
-
-	pkgmap := enumeratePackages(t)
-	for dir, pkg := range pkgmap {
-		// Write a "main_test.go" file to the package.
-		// This file is required to run the integration tests.
-		f, err := os.Create(filepath.Join(dir, "main_test.go"))
-		require.NoError(t, err)
-		defer f.Close()
-		err = mainTestTemplate.Execute(f, pkg)
-		require.NoError(t, err)
-	}
-}
diff --git a/integration/internal/main.go b/integration/internal/main.go
deleted file mode 100644
index 6aa2a4c93..000000000
--- a/integration/internal/main.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package internal
-
-import (
-	"fmt"
-	"os"
-	"testing"
-
-	"github.com/databricks/cli/integration/internal/acc"
-)
-
-// Main is the entry point for integration tests.
-// We use this for all integration tests defined in this subtree to ensure
-// they are not inadvertently executed when calling `go test ./...`.
-func Main(m *testing.M) {
-	value := os.Getenv("CLOUD_ENV")
-	if value == "" && !acc.IsInDebug() {
-		fmt.Println("CLOUD_ENV is not set, skipping integration tests")
-		return
-	}
-
-	m.Run()
-}
diff --git a/integration/libs/filer/main_test.go b/integration/libs/filer/main_test.go
deleted file mode 100644
index ca866d952..000000000
--- a/integration/libs/filer/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package filer_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
diff --git a/integration/libs/git/main_test.go b/integration/libs/git/main_test.go
deleted file mode 100644
index 5d68e0851..000000000
--- a/integration/libs/git/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package git_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
diff --git a/integration/libs/locker/main_test.go b/integration/libs/locker/main_test.go
deleted file mode 100644
index 33a883768..000000000
--- a/integration/libs/locker/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package locker_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
diff --git a/integration/libs/tags/main_test.go b/integration/libs/tags/main_test.go
deleted file mode 100644
index 4eaf54a20..000000000
--- a/integration/libs/tags/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package tags_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}
diff --git a/integration/python/main_test.go b/integration/python/main_test.go
deleted file mode 100644
index b35da21e1..000000000
--- a/integration/python/main_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package python_test
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/integration/internal"
-)
-
-// TestMain is the entrypoint executed by the test runner.
-// See [internal.Main] for prerequisites for running integration tests.
-func TestMain(m *testing.M) {
-	internal.Main(m)
-}

From 185bbd28e476206da31c0db2d17b9e3e89d6697b Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 8 Jan 2025 13:41:08 +0100
Subject: [PATCH 034/247] Add acceptance tests (#2081)

## Changes
- New kind of test is added - acceptance tests. See acceptance/README.md
for explanation.
- A few tests are converted to acceptance tests by moving databricks.yml
to acceptance/ and adding corresponding script files.

As these tests run against compiled binary and can capture full output
of the command, they can be useful to support major changes such as
refactoring internal logging / diagnostics or complex variable
interpolation.

These are currently run as part of 'make test' but the intention is to
run them as part of integration tests as well.

### Benefits

- Full binary is tested, exactly as users get it.
  - We're not testing custom set of mutators like many existing tests.
- Not mocking anything, real SDK is used (although the HTTP endpoint is
not a real Databricks env).
- Easy to maintain: output can be updated automatically.
- Can easily set up external env, such as env vars, CLI args,
.databrickscfg location etc.

### Gaps

The tests currently share the test server and there is global place to
define handlers. We should have a way for tests to override / add new
handlers.

## Tests
I manually checked that output of new acceptance tests matches previous
asserts.
---
 Makefile                                      |   2 +-
 acceptance/README.md                          |  19 ++
 acceptance/acceptance_test.go                 | 302 ++++++++++++++++++
 acceptance/build/.gitignore                   |   1 +
 .../bundle/override}/clusters/databricks.yml  |   3 -
 .../bundle/override/clusters/output.txt       |  33 ++
 acceptance/bundle/override/clusters/script    |   2 +
 .../override/job_cluster}/databricks.yml      |   3 -
 .../bundle/override/job_cluster/output.txt    |  30 ++
 acceptance/bundle/override/job_cluster/script |   2 +
 .../bundle/override/job_tasks}/databricks.yml |   3 -
 .../bundle/override/job_tasks/output.txt      |  68 ++++
 acceptance/bundle/override/job_tasks/script   |   2 +
 .../override/merge-string-map/databricks.yml  |  13 +
 .../override/merge-string-map/output.txt      |  23 ++
 .../bundle/override/merge-string-map/script   |   2 +
 .../override/pipeline_cluster}/databricks.yml |   3 -
 .../override/pipeline_cluster/output.txt      |  44 +++
 .../bundle/override/pipeline_cluster/script   |   2 +
 .../complex-transitive/databricks.yml         |  19 ++
 .../variables/complex-transitive/output.txt   |   3 +
 .../variables/complex-transitive/script       |   2 +
 .../bundle}/variables/complex/databricks.yml  |   1 +
 .../bundle/variables/complex/out.default.json | 124 +++++++
 .../bundle/variables/complex/out.dev.json     | 118 +++++++
 acceptance/bundle/variables/complex/script    |   4 +
 .../complex_multiple_files/databricks.yml     |   0
 .../complex_multiple_files/output.txt         | 159 +++++++++
 .../variables/complex_multiple_files/script   |   1 +
 .../variables/clusters.yml                    |   0
 .../bundle}/variables/empty/databricks.yml    |   0
 acceptance/bundle/variables/empty/output.txt  |  11 +
 acceptance/bundle/variables/empty/script      |   1 +
 .../variables/env_overrides/databricks.yml    |   0
 .../bundle/variables/env_overrides/output.txt |  40 +++
 .../bundle/variables/env_overrides/script     |   6 +
 .../bundle}/variables/vanilla/databricks.yml  |   0
 .../bundle/variables/vanilla/output.txt       |  16 +
 acceptance/bundle/variables/vanilla/script    |   2 +
 .../databricks.yml                            |   0
 .../variable_overrides_in_target/output.txt   |  84 +++++
 .../variable_overrides_in_target/script       |   4 +
 .../without_definition/databricks.yml         |   2 +
 .../variables/without_definition/output.txt   |   4 +
 .../variables/without_definition/script       |   1 +
 acceptance/help/output.txt                    | 143 +++++++++
 acceptance/help/script                        |   1 +
 acceptance/script.cleanup                     |   1 +
 acceptance/script.prepare                     |  36 +++
 acceptance/server_test.go                     | 129 ++++++++
 bundle/tests/clusters_test.go                 |  36 ---
 bundle/tests/complex_variables_test.go        | 108 -------
 bundle/tests/override_job_cluster_test.go     |  29 --
 bundle/tests/override_job_tasks_test.go       |  39 ---
 .../tests/override_pipeline_cluster_test.go   |  29 --
 bundle/tests/variables_test.go                | 206 ------------
 libs/env/context.go                           |   6 +-
 libs/testdiff/testdiff.go                     |   4 +-
 58 files changed, 1462 insertions(+), 464 deletions(-)
 create mode 100644 acceptance/README.md
 create mode 100644 acceptance/acceptance_test.go
 create mode 100644 acceptance/build/.gitignore
 rename {bundle/tests => acceptance/bundle/override}/clusters/databricks.yml (92%)
 create mode 100644 acceptance/bundle/override/clusters/output.txt
 create mode 100644 acceptance/bundle/override/clusters/script
 rename {bundle/tests/override_job_cluster => acceptance/bundle/override/job_cluster}/databricks.yml (91%)
 create mode 100644 acceptance/bundle/override/job_cluster/output.txt
 create mode 100644 acceptance/bundle/override/job_cluster/script
 rename {bundle/tests/override_job_tasks => acceptance/bundle/override/job_tasks}/databricks.yml (94%)
 create mode 100644 acceptance/bundle/override/job_tasks/output.txt
 create mode 100644 acceptance/bundle/override/job_tasks/script
 create mode 100644 acceptance/bundle/override/merge-string-map/databricks.yml
 create mode 100644 acceptance/bundle/override/merge-string-map/output.txt
 create mode 100644 acceptance/bundle/override/merge-string-map/script
 rename {bundle/tests/override_pipeline_cluster => acceptance/bundle/override/pipeline_cluster}/databricks.yml (90%)
 create mode 100644 acceptance/bundle/override/pipeline_cluster/output.txt
 create mode 100644 acceptance/bundle/override/pipeline_cluster/script
 create mode 100644 acceptance/bundle/variables/complex-transitive/databricks.yml
 create mode 100644 acceptance/bundle/variables/complex-transitive/output.txt
 create mode 100644 acceptance/bundle/variables/complex-transitive/script
 rename {bundle/tests => acceptance/bundle}/variables/complex/databricks.yml (98%)
 create mode 100644 acceptance/bundle/variables/complex/out.default.json
 create mode 100644 acceptance/bundle/variables/complex/out.dev.json
 create mode 100644 acceptance/bundle/variables/complex/script
 rename {bundle/tests => acceptance/bundle}/variables/complex_multiple_files/databricks.yml (100%)
 create mode 100644 acceptance/bundle/variables/complex_multiple_files/output.txt
 create mode 100644 acceptance/bundle/variables/complex_multiple_files/script
 rename {bundle/tests => acceptance/bundle}/variables/complex_multiple_files/variables/clusters.yml (100%)
 rename {bundle/tests => acceptance/bundle}/variables/empty/databricks.yml (100%)
 create mode 100644 acceptance/bundle/variables/empty/output.txt
 create mode 100644 acceptance/bundle/variables/empty/script
 rename {bundle/tests => acceptance/bundle}/variables/env_overrides/databricks.yml (100%)
 create mode 100644 acceptance/bundle/variables/env_overrides/output.txt
 create mode 100644 acceptance/bundle/variables/env_overrides/script
 rename {bundle/tests => acceptance/bundle}/variables/vanilla/databricks.yml (100%)
 create mode 100644 acceptance/bundle/variables/vanilla/output.txt
 create mode 100644 acceptance/bundle/variables/vanilla/script
 rename {bundle/tests => acceptance/bundle}/variables/variable_overrides_in_target/databricks.yml (100%)
 create mode 100644 acceptance/bundle/variables/variable_overrides_in_target/output.txt
 create mode 100644 acceptance/bundle/variables/variable_overrides_in_target/script
 rename {bundle/tests => acceptance/bundle}/variables/without_definition/databricks.yml (53%)
 create mode 100644 acceptance/bundle/variables/without_definition/output.txt
 create mode 100644 acceptance/bundle/variables/without_definition/script
 create mode 100644 acceptance/help/output.txt
 create mode 100644 acceptance/help/script
 create mode 100644 acceptance/script.cleanup
 create mode 100644 acceptance/script.prepare
 create mode 100644 acceptance/server_test.go
 delete mode 100644 bundle/tests/clusters_test.go
 delete mode 100644 bundle/tests/complex_variables_test.go
 delete mode 100644 bundle/tests/override_job_cluster_test.go
 delete mode 100644 bundle/tests/override_job_tasks_test.go
 delete mode 100644 bundle/tests/override_pipeline_cluster_test.go
 delete mode 100644 bundle/tests/variables_test.go

diff --git a/Makefile b/Makefile
index f8b725900..40eef9f31 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 default: build
 
-PACKAGES=./libs/... ./internal/... ./cmd/... ./bundle/... .
+PACKAGES=./acceptance/... ./libs/... ./internal/... ./cmd/... ./bundle/... .
 
 GOTESTSUM_FORMAT ?= pkgname-and-test-fails
 
diff --git a/acceptance/README.md b/acceptance/README.md
new file mode 100644
index 000000000..162c57ea2
--- /dev/null
+++ b/acceptance/README.md
@@ -0,0 +1,19 @@
+Acceptance tests are blackbox tests that are run against compiled binary.
+
+Currently these tests are run against "fake" HTTP server pretending to be Databricks API. However, they will be extended to run against real environment as regular integration tests.
+
+To author a test,
+ - Add a new directory under `acceptance`. Any level of nesting is supported.
+ - Add `databricks.yml` there.
+ - Add `script` with commands to run, e.g. `$CLI bundle validate`. The test case is recognized by presence of `script`.
+
+The test runner will run script and capture output and compare it with `output.txt` file in the same directory.
+
+In order to write `output.txt` for the first time or overwrite it with the current output, set `TESTS_OUTPUT=OVERWRITE` env var.
+
+The scripts are run with `bash -e` so any errors will be propagated. They are captured in `output.txt` by appending `Exit code: N` line at the end.
+
+For more complex tests one can also use:
+- `errcode` helper: if the command fails with non-zero code, it appends `Exit code: N` to the output but returns success to caller (bash), allowing continuation of script.
+- `trace` helper: prints the arguments before executing the command.
+- custom output files: redirect output to custom file (it must start with `out`), e.g. `$CLI bundle validate > out.txt 2> out.error.txt`.
diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
new file mode 100644
index 000000000..759c0aeca
--- /dev/null
+++ b/acceptance/acceptance_test.go
@@ -0,0 +1,302 @@
+package acceptance_test
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"slices"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/databricks/cli/internal/testutil"
+	"github.com/databricks/cli/libs/env"
+	"github.com/databricks/cli/libs/testdiff"
+	"github.com/stretchr/testify/require"
+)
+
+var KeepTmp = os.Getenv("KEEP_TMP") != ""
+
+const (
+	EntryPointScript = "script"
+	CleanupScript    = "script.cleanup"
+	PrepareScript    = "script.prepare"
+)
+
+var Scripts = map[string]bool{
+	EntryPointScript: true,
+	CleanupScript:    true,
+	PrepareScript:    true,
+}
+
+func TestAccept(t *testing.T) {
+	execPath := BuildCLI(t)
+	// $CLI is what test scripts are using
+	t.Setenv("CLI", execPath)
+
+	server := StartServer(t)
+	AddHandlers(server)
+	// Redirect API access to local server:
+	t.Setenv("DATABRICKS_HOST", fmt.Sprintf("http://127.0.0.1:%d", server.Port))
+	t.Setenv("DATABRICKS_TOKEN", "dapi1234")
+
+	homeDir := t.TempDir()
+	// Do not read user's ~/.databrickscfg
+	t.Setenv(env.HomeEnvVar(), homeDir)
+
+	testDirs := getTests(t)
+	require.NotEmpty(t, testDirs)
+	for _, dir := range testDirs {
+		t.Run(dir, func(t *testing.T) {
+			t.Parallel()
+			runTest(t, dir)
+		})
+	}
+}
+
+func getTests(t *testing.T) []string {
+	testDirs := make([]string, 0, 128)
+
+	err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		name := filepath.Base(path)
+		if name == EntryPointScript {
+			// Presence of 'script' marks a test case in this directory
+			testDirs = append(testDirs, filepath.Dir(path))
+		}
+		return nil
+	})
+	require.NoError(t, err)
+
+	sort.Strings(testDirs)
+	return testDirs
+}
+
+func runTest(t *testing.T, dir string) {
+	var tmpDir string
+	var err error
+	if KeepTmp {
+		tempDirBase := filepath.Join(os.TempDir(), "acceptance")
+		_ = os.Mkdir(tempDirBase, 0o755)
+		tmpDir, err = os.MkdirTemp(tempDirBase, "")
+		require.NoError(t, err)
+		t.Logf("Created directory: %s", tmpDir)
+	} else {
+		tmpDir = t.TempDir()
+	}
+
+	scriptContents := readMergedScriptContents(t, dir)
+	testutil.WriteFile(t, filepath.Join(tmpDir, EntryPointScript), scriptContents)
+
+	inputs := make(map[string]bool, 2)
+	outputs := make(map[string]bool, 2)
+	err = CopyDir(dir, tmpDir, inputs, outputs)
+	require.NoError(t, err)
+
+	args := []string{"bash", "-euo", "pipefail", EntryPointScript}
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Dir = tmpDir
+	outB, err := cmd.CombinedOutput()
+
+	out := formatOutput(string(outB), err)
+	out = strings.ReplaceAll(out, os.Getenv("CLI"), "$CLI")
+	doComparison(t, filepath.Join(dir, "output.txt"), "script output", out)
+
+	for key := range outputs {
+		if key == "output.txt" {
+			// handled above
+			continue
+		}
+		pathNew := filepath.Join(tmpDir, key)
+		newValBytes, err := os.ReadFile(pathNew)
+		if err != nil {
+			if errors.Is(err, os.ErrNotExist) {
+				t.Errorf("%s: expected to find this file but could not (%s)", key, tmpDir)
+			} else {
+				t.Errorf("%s: could not read: %s", key, err)
+			}
+			continue
+		}
+		pathExpected := filepath.Join(dir, key)
+		doComparison(t, pathExpected, pathNew, string(newValBytes))
+	}
+
+	// Make sure there are not unaccounted for new files
+	files, err := os.ReadDir(tmpDir)
+	require.NoError(t, err)
+
+	for _, f := range files {
+		name := f.Name()
+		if _, ok := inputs[name]; ok {
+			continue
+		}
+		if _, ok := outputs[name]; ok {
+			continue
+		}
+		t.Errorf("Unexpected output: %s", f)
+		if strings.HasPrefix(name, "out") {
+			// We have a new file starting with "out"
+			// Show the contents & support overwrite mode for it:
+			pathNew := filepath.Join(tmpDir, name)
+			newVal := testutil.ReadFile(t, pathNew)
+			doComparison(t, filepath.Join(dir, name), filepath.Join(tmpDir, name), newVal)
+		}
+	}
+}
+
+func doComparison(t *testing.T, pathExpected, pathNew, valueNew string) {
+	valueNew = testdiff.NormalizeNewlines(valueNew)
+	valueExpected := string(readIfExists(t, pathExpected))
+	valueExpected = testdiff.NormalizeNewlines(valueExpected)
+	testdiff.AssertEqualTexts(t, pathExpected, pathNew, valueExpected, valueNew)
+	if testdiff.OverwriteMode {
+		if valueNew != "" {
+			t.Logf("Overwriting: %s", pathExpected)
+			testutil.WriteFile(t, pathExpected, valueNew)
+		} else {
+			t.Logf("Removing: %s", pathExpected)
+			_ = os.Remove(pathExpected)
+		}
+	}
+}
+
+// Returns combined script.prepare (root) + script.prepare (parent) + ... + script + ... + script.cleanup (parent) + ...
+// Note, cleanups are not executed if main script fails; that's not a huge issue, since it runs it temp dir.
+func readMergedScriptContents(t *testing.T, dir string) string {
+	scriptContents := testutil.ReadFile(t, filepath.Join(dir, EntryPointScript))
+	prepares := []string{}
+	cleanups := []string{}
+
+	for {
+		x := readIfExists(t, filepath.Join(dir, CleanupScript))
+		if len(x) > 0 {
+			cleanups = append(cleanups, string(x))
+		}
+
+		x = readIfExists(t, filepath.Join(dir, PrepareScript))
+		if len(x) > 0 {
+			prepares = append(prepares, string(x))
+		}
+
+		if dir == "" || dir == "." {
+			break
+		}
+
+		dir = filepath.Dir(dir)
+		require.True(t, filepath.IsLocal(dir))
+	}
+
+	slices.Reverse(prepares)
+	prepares = append(prepares, scriptContents)
+	prepares = append(prepares, cleanups...)
+	return strings.Join(prepares, "\n")
+}
+
+func BuildCLI(t *testing.T) string {
+	cwd, err := os.Getwd()
+	require.NoError(t, err)
+	execPath := filepath.Join(cwd, "build", "databricks")
+	if runtime.GOOS == "windows" {
+		execPath += ".exe"
+	}
+
+	start := time.Now()
+	args := []string{"go", "build", "-mod", "vendor", "-o", execPath}
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Dir = ".."
+	out, err := cmd.CombinedOutput()
+	elapsed := time.Since(start)
+	t.Logf("%s took %s", args, elapsed)
+	require.NoError(t, err, "go build failed: %s: %s\n%s", args, err, out)
+	if len(out) > 0 {
+		t.Logf("go build output: %s: %s", args, out)
+	}
+
+	// Quick check + warm up cache:
+	cmd = exec.Command(execPath, "--version")
+	out, err = cmd.CombinedOutput()
+	require.NoError(t, err, "%s --version failed: %s\n%s", execPath, err, out)
+	return execPath
+}
+
+func copyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, in)
+	return err
+}
+
+func formatOutput(out string, err error) string {
+	if err == nil {
+		return out
+	}
+	if exiterr, ok := err.(*exec.ExitError); ok {
+		exitCode := exiterr.ExitCode()
+		out += fmt.Sprintf("\nExit code: %d\n", exitCode)
+	} else {
+		out += fmt.Sprintf("\nError: %s\n", err)
+	}
+	return out
+}
+
+func readIfExists(t *testing.T, path string) []byte {
+	data, err := os.ReadFile(path)
+	if err == nil {
+		return data
+	}
+
+	if !errors.Is(err, os.ErrNotExist) {
+		t.Fatalf("%s: %s", path, err)
+	}
+	return []byte{}
+}
+
+func CopyDir(src, dst string, inputs, outputs map[string]bool) error {
+	return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		name := info.Name()
+
+		relPath, err := filepath.Rel(src, path)
+		if err != nil {
+			return err
+		}
+
+		if strings.HasPrefix(name, "out") {
+			outputs[relPath] = true
+			return nil
+		} else {
+			inputs[relPath] = true
+		}
+
+		if _, ok := Scripts[name]; ok {
+			return nil
+		}
+
+		destPath := filepath.Join(dst, relPath)
+
+		if info.IsDir() {
+			return os.MkdirAll(destPath, info.Mode())
+		}
+
+		return copyFile(path, destPath)
+	})
+}
diff --git a/acceptance/build/.gitignore b/acceptance/build/.gitignore
new file mode 100644
index 000000000..a48b4db25
--- /dev/null
+++ b/acceptance/build/.gitignore
@@ -0,0 +1 @@
+databricks
diff --git a/bundle/tests/clusters/databricks.yml b/acceptance/bundle/override/clusters/databricks.yml
similarity index 92%
rename from bundle/tests/clusters/databricks.yml
rename to acceptance/bundle/override/clusters/databricks.yml
index 1074462a6..14efceec0 100644
--- a/bundle/tests/clusters/databricks.yml
+++ b/acceptance/bundle/override/clusters/databricks.yml
@@ -1,9 +1,6 @@
 bundle:
   name: clusters
 
-workspace:
-  host: https://acme.cloud.databricks.com/
-
 resources:
   clusters:
     foo:
diff --git a/acceptance/bundle/override/clusters/output.txt b/acceptance/bundle/override/clusters/output.txt
new file mode 100644
index 000000000..cff30b3af
--- /dev/null
+++ b/acceptance/bundle/override/clusters/output.txt
@@ -0,0 +1,33 @@
+
+>>> $CLI bundle validate -o json -t default
+{
+  "autoscale": {
+    "max_workers": 7,
+    "min_workers": 2
+  },
+  "cluster_name": "foo",
+  "custom_tags": {},
+  "node_type_id": "i3.xlarge",
+  "num_workers": 2,
+  "spark_conf": {
+    "spark.executor.memory": "2g"
+  },
+  "spark_version": "13.3.x-scala2.12"
+}
+
+>>> $CLI bundle validate -o json -t development
+{
+  "autoscale": {
+    "max_workers": 3,
+    "min_workers": 1
+  },
+  "cluster_name": "foo-override",
+  "custom_tags": {},
+  "node_type_id": "m5.xlarge",
+  "num_workers": 3,
+  "spark_conf": {
+    "spark.executor.memory": "4g",
+    "spark.executor.memory2": "4g"
+  },
+  "spark_version": "15.2.x-scala2.12"
+}
diff --git a/acceptance/bundle/override/clusters/script b/acceptance/bundle/override/clusters/script
new file mode 100644
index 000000000..4a73dd93e
--- /dev/null
+++ b/acceptance/bundle/override/clusters/script
@@ -0,0 +1,2 @@
+trace $CLI bundle validate -o json -t default | jq .resources.clusters.foo
+trace $CLI bundle validate -o json -t development | jq .resources.clusters.foo
diff --git a/bundle/tests/override_job_cluster/databricks.yml b/acceptance/bundle/override/job_cluster/databricks.yml
similarity index 91%
rename from bundle/tests/override_job_cluster/databricks.yml
rename to acceptance/bundle/override/job_cluster/databricks.yml
index a85b3b711..d6b7ede4f 100644
--- a/bundle/tests/override_job_cluster/databricks.yml
+++ b/acceptance/bundle/override/job_cluster/databricks.yml
@@ -1,9 +1,6 @@
 bundle:
   name: override_job_cluster
 
-workspace:
-  host: https://acme.cloud.databricks.com/
-
 resources:
   jobs:
     foo:
diff --git a/acceptance/bundle/override/job_cluster/output.txt b/acceptance/bundle/override/job_cluster/output.txt
new file mode 100644
index 000000000..dc7a5f75b
--- /dev/null
+++ b/acceptance/bundle/override/job_cluster/output.txt
@@ -0,0 +1,30 @@
+
+>>> $CLI bundle validate -o json -t development
+{
+  "name": "job",
+  "job_clusters": [
+    {
+      "job_cluster_key": "key",
+      "new_cluster": {
+        "node_type_id": "i3.xlarge",
+        "num_workers": 1,
+        "spark_version": "13.3.x-scala2.12"
+      }
+    }
+  ]
+}
+
+>>> $CLI bundle validate -o json -t staging
+{
+  "name": "job",
+  "job_clusters": [
+    {
+      "job_cluster_key": "key",
+      "new_cluster": {
+        "node_type_id": "i3.2xlarge",
+        "num_workers": 4,
+        "spark_version": "13.3.x-scala2.12"
+      }
+    }
+  ]
+}
diff --git a/acceptance/bundle/override/job_cluster/script b/acceptance/bundle/override/job_cluster/script
new file mode 100644
index 000000000..3f6827bb7
--- /dev/null
+++ b/acceptance/bundle/override/job_cluster/script
@@ -0,0 +1,2 @@
+trace $CLI bundle validate -o json -t development | jq '.resources.jobs.foo | {name,job_clusters}'
+trace $CLI bundle validate -o json -t staging | jq '.resources.jobs.foo | {name,job_clusters}'
diff --git a/bundle/tests/override_job_tasks/databricks.yml b/acceptance/bundle/override/job_tasks/databricks.yml
similarity index 94%
rename from bundle/tests/override_job_tasks/databricks.yml
rename to acceptance/bundle/override/job_tasks/databricks.yml
index ddee28793..fd7edafb9 100644
--- a/bundle/tests/override_job_tasks/databricks.yml
+++ b/acceptance/bundle/override/job_tasks/databricks.yml
@@ -1,9 +1,6 @@
 bundle:
   name: override_job_tasks
 
-workspace:
-  host: https://acme.cloud.databricks.com/
-
 resources:
   jobs:
     foo:
diff --git a/acceptance/bundle/override/job_tasks/output.txt b/acceptance/bundle/override/job_tasks/output.txt
new file mode 100644
index 000000000..0d561291e
--- /dev/null
+++ b/acceptance/bundle/override/job_tasks/output.txt
@@ -0,0 +1,68 @@
+
+>>> errcode $CLI bundle validate -o json -t development
+Error: file ./test1.py not found
+
+Exit code: 1
+{
+  "name": "job",
+  "queue": {
+    "enabled": true
+  },
+  "tags": {},
+  "tasks": [
+    {
+      "new_cluster": {
+        "node_type_id": "i3.xlarge",
+        "num_workers": 1,
+        "spark_version": "13.3.x-scala2.12"
+      },
+      "spark_python_task": {
+        "python_file": "./test1.py"
+      },
+      "task_key": "key1"
+    },
+    {
+      "new_cluster": {
+        "spark_version": "13.3.x-scala2.12"
+      },
+      "spark_python_task": {
+        "python_file": "./test2.py"
+      },
+      "task_key": "key2"
+    }
+  ]
+}
+
+>>> errcode $CLI bundle validate -o json -t staging
+Error: file ./test1.py not found
+
+Exit code: 1
+{
+  "name": "job",
+  "queue": {
+    "enabled": true
+  },
+  "tags": {},
+  "tasks": [
+    {
+      "new_cluster": {
+        "spark_version": "13.3.x-scala2.12"
+      },
+      "spark_python_task": {
+        "python_file": "./test1.py"
+      },
+      "task_key": "key1"
+    },
+    {
+      "new_cluster": {
+        "node_type_id": "i3.2xlarge",
+        "num_workers": 4,
+        "spark_version": "13.3.x-scala2.12"
+      },
+      "spark_python_task": {
+        "python_file": "./test3.py"
+      },
+      "task_key": "key2"
+    }
+  ]
+}
diff --git a/acceptance/bundle/override/job_tasks/script b/acceptance/bundle/override/job_tasks/script
new file mode 100644
index 000000000..4e0869857
--- /dev/null
+++ b/acceptance/bundle/override/job_tasks/script
@@ -0,0 +1,2 @@
+trace errcode $CLI bundle validate -o json -t development | jq .resources.jobs.foo
+trace errcode $CLI bundle validate -o json -t staging | jq .resources.jobs.foo
diff --git a/acceptance/bundle/override/merge-string-map/databricks.yml b/acceptance/bundle/override/merge-string-map/databricks.yml
new file mode 100644
index 000000000..5e443ceca
--- /dev/null
+++ b/acceptance/bundle/override/merge-string-map/databricks.yml
@@ -0,0 +1,13 @@
+bundle:
+  name: merge-string-map
+
+resources:
+  clusters:
+    my_cluster: "hello"
+
+targets:
+  dev:
+    resources:
+      clusters:
+        my_cluster:
+          spark_version: "25"
diff --git a/acceptance/bundle/override/merge-string-map/output.txt b/acceptance/bundle/override/merge-string-map/output.txt
new file mode 100644
index 000000000..e1bd7dfb4
--- /dev/null
+++ b/acceptance/bundle/override/merge-string-map/output.txt
@@ -0,0 +1,23 @@
+
+>>> $CLI bundle validate -o json -t dev
+{
+  "clusters": {
+    "my_cluster": {
+      "custom_tags": {},
+      "spark_version": "25"
+    }
+  }
+}
+
+>>> $CLI bundle validate -t dev
+Warning: expected map, found string
+  at resources.clusters.my_cluster
+  in databricks.yml:6:17
+
+Name: merge-string-map
+Target: dev
+Workspace:
+  User: tester@databricks.com
+  Path: /Workspace/Users/tester@databricks.com/.bundle/merge-string-map/dev
+
+Found 1 warning
diff --git a/acceptance/bundle/override/merge-string-map/script b/acceptance/bundle/override/merge-string-map/script
new file mode 100644
index 000000000..a109d5f69
--- /dev/null
+++ b/acceptance/bundle/override/merge-string-map/script
@@ -0,0 +1,2 @@
+trace $CLI bundle validate -o json -t dev | jq .resources
+trace $CLI bundle validate -t dev
diff --git a/bundle/tests/override_pipeline_cluster/databricks.yml b/acceptance/bundle/override/pipeline_cluster/databricks.yml
similarity index 90%
rename from bundle/tests/override_pipeline_cluster/databricks.yml
rename to acceptance/bundle/override/pipeline_cluster/databricks.yml
index 8930f30e8..8b4857460 100644
--- a/bundle/tests/override_pipeline_cluster/databricks.yml
+++ b/acceptance/bundle/override/pipeline_cluster/databricks.yml
@@ -1,9 +1,6 @@
 bundle:
   name: override_pipeline_cluster
 
-workspace:
-  host: https://acme.cloud.databricks.com/
-
 resources:
   pipelines:
     foo:
diff --git a/acceptance/bundle/override/pipeline_cluster/output.txt b/acceptance/bundle/override/pipeline_cluster/output.txt
new file mode 100644
index 000000000..81bf58180
--- /dev/null
+++ b/acceptance/bundle/override/pipeline_cluster/output.txt
@@ -0,0 +1,44 @@
+
+>>> $CLI bundle validate -o json -t development
+{
+  "foo": {
+    "clusters": [
+      {
+        "label": "default",
+        "node_type_id": "i3.xlarge",
+        "num_workers": 1,
+        "spark_conf": {
+          "foo": "bar"
+        }
+      }
+    ],
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_pipeline_cluster/development/state/metadata.json"
+    },
+    "name": "job",
+    "permissions": []
+  }
+}
+
+>>> $CLI bundle validate -o json -t staging
+{
+  "foo": {
+    "clusters": [
+      {
+        "label": "default",
+        "node_type_id": "i3.2xlarge",
+        "num_workers": 4,
+        "spark_conf": {
+          "foo": "bar"
+        }
+      }
+    ],
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_pipeline_cluster/staging/state/metadata.json"
+    },
+    "name": "job",
+    "permissions": []
+  }
+}
diff --git a/acceptance/bundle/override/pipeline_cluster/script b/acceptance/bundle/override/pipeline_cluster/script
new file mode 100644
index 000000000..b06005ce5
--- /dev/null
+++ b/acceptance/bundle/override/pipeline_cluster/script
@@ -0,0 +1,2 @@
+trace $CLI bundle validate -o json -t development | jq .resources.pipelines
+trace $CLI bundle validate -o json -t staging | jq .resources.pipelines
diff --git a/acceptance/bundle/variables/complex-transitive/databricks.yml b/acceptance/bundle/variables/complex-transitive/databricks.yml
new file mode 100644
index 000000000..9ef4e6386
--- /dev/null
+++ b/acceptance/bundle/variables/complex-transitive/databricks.yml
@@ -0,0 +1,19 @@
+bundle:
+  name: complex-transitive
+
+variables:
+  catalog:
+    default: hive_metastore
+  spark_conf:
+    default:
+      "spark.databricks.sql.initial.catalog.name": ${var.catalog}
+  etl_cluster_config:
+    type: complex
+    default:
+      spark_version: 14.3.x-scala2.12
+      runtime_engine: PHOTON
+      spark_conf: ${var.spark_conf}
+
+resources:
+  clusters:
+    my_cluster: ${var.etl_cluster_config}
diff --git a/acceptance/bundle/variables/complex-transitive/output.txt b/acceptance/bundle/variables/complex-transitive/output.txt
new file mode 100644
index 000000000..a031e0497
--- /dev/null
+++ b/acceptance/bundle/variables/complex-transitive/output.txt
@@ -0,0 +1,3 @@
+{
+  "spark.databricks.sql.initial.catalog.name": "${var.catalog}"
+}
diff --git a/acceptance/bundle/variables/complex-transitive/script b/acceptance/bundle/variables/complex-transitive/script
new file mode 100644
index 000000000..52bb08ed4
--- /dev/null
+++ b/acceptance/bundle/variables/complex-transitive/script
@@ -0,0 +1,2 @@
+# Currently, this incorrectly outputs variable reference instead of resolved value
+$CLI bundle validate -o json | jq '.resources.clusters.my_cluster.spark_conf'
diff --git a/bundle/tests/variables/complex/databricks.yml b/acceptance/bundle/variables/complex/databricks.yml
similarity index 98%
rename from bundle/tests/variables/complex/databricks.yml
rename to acceptance/bundle/variables/complex/databricks.yml
index 3b32a7c8e..500f374e3 100644
--- a/bundle/tests/variables/complex/databricks.yml
+++ b/acceptance/bundle/variables/complex/databricks.yml
@@ -46,6 +46,7 @@ variables:
 
 targets:
   default:
+    default: true
   dev:
     variables:
       node_type: "Standard_DS3_v3"
diff --git a/acceptance/bundle/variables/complex/out.default.json b/acceptance/bundle/variables/complex/out.default.json
new file mode 100644
index 000000000..bbdb0f8de
--- /dev/null
+++ b/acceptance/bundle/variables/complex/out.default.json
@@ -0,0 +1,124 @@
+{
+  "resources": {
+    "jobs": {
+      "my_job": {
+        "deployment": {
+          "kind": "BUNDLE",
+          "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/complex-variables/default/state/metadata.json"
+        },
+        "edit_mode": "UI_LOCKED",
+        "format": "MULTI_TASK",
+        "job_clusters": [
+          {
+            "job_cluster_key": "key",
+            "new_cluster": {
+              "node_type_id": "Standard_DS3_v2",
+              "num_workers": 2,
+              "policy_id": "some-policy-id",
+              "spark_conf": {
+                "spark.databricks.delta.retentionDurationCheck.enabled": "false",
+                "spark.random": "true",
+                "spark.speculation": "true"
+              },
+              "spark_version": "13.2.x-scala2.11"
+            }
+          }
+        ],
+        "permissions": [],
+        "queue": {
+          "enabled": true
+        },
+        "tags": {},
+        "tasks": [
+          {
+            "job_cluster_key": "key",
+            "libraries": [
+              {
+                "jar": "/path/to/jar"
+              },
+              {
+                "egg": "/path/to/egg"
+              },
+              {
+                "whl": "/path/to/whl"
+              }
+            ],
+            "task_key": "task with spark version 13.2.x-scala2.11 and jar /path/to/jar"
+          }
+        ]
+      }
+    }
+  },
+  "variables": {
+    "cluster": {
+      "default": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 2,
+        "policy_id": "some-policy-id",
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.random": true,
+          "spark.speculation": true
+        },
+        "spark_version": "13.2.x-scala2.11"
+      },
+      "description": "A cluster definition",
+      "type": "complex",
+      "value": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 2,
+        "policy_id": "some-policy-id",
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.random": true,
+          "spark.speculation": true
+        },
+        "spark_version": "13.2.x-scala2.11"
+      }
+    },
+    "complexvar": {
+      "default": {
+        "key1": "value1",
+        "key2": "value2",
+        "key3": "value3"
+      },
+      "description": "A complex variable",
+      "type": "complex",
+      "value": {
+        "key1": "value1",
+        "key2": "value2",
+        "key3": "value3"
+      }
+    },
+    "libraries": {
+      "default": [
+        {
+          "jar": "/path/to/jar"
+        },
+        {
+          "egg": "/path/to/egg"
+        },
+        {
+          "whl": "/path/to/whl"
+        }
+      ],
+      "description": "A libraries definition",
+      "type": "complex",
+      "value": [
+        {
+          "jar": "/path/to/jar"
+        },
+        {
+          "egg": "/path/to/egg"
+        },
+        {
+          "whl": "/path/to/whl"
+        }
+      ]
+    },
+    "node_type": {
+      "default": "Standard_DS3_v2",
+      "value": "Standard_DS3_v2"
+    }
+  }
+}
diff --git a/acceptance/bundle/variables/complex/out.dev.json b/acceptance/bundle/variables/complex/out.dev.json
new file mode 100644
index 000000000..330518c85
--- /dev/null
+++ b/acceptance/bundle/variables/complex/out.dev.json
@@ -0,0 +1,118 @@
+{
+  "resources": {
+    "jobs": {
+      "my_job": {
+        "deployment": {
+          "kind": "BUNDLE",
+          "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/complex-variables/dev/state/metadata.json"
+        },
+        "edit_mode": "UI_LOCKED",
+        "format": "MULTI_TASK",
+        "job_clusters": [
+          {
+            "job_cluster_key": "key",
+            "new_cluster": {
+              "node_type_id": "Standard_DS3_v3",
+              "num_workers": 4,
+              "spark_conf": {
+                "spark.databricks.delta.retentionDurationCheck.enabled": "false",
+                "spark.speculation": "false"
+              },
+              "spark_version": "14.2.x-scala2.11"
+            }
+          }
+        ],
+        "permissions": [],
+        "queue": {
+          "enabled": true
+        },
+        "tags": {},
+        "tasks": [
+          {
+            "job_cluster_key": "key",
+            "libraries": [
+              {
+                "jar": "/path/to/jar"
+              },
+              {
+                "egg": "/path/to/egg"
+              },
+              {
+                "whl": "/path/to/whl"
+              }
+            ],
+            "task_key": "task with spark version 14.2.x-scala2.11 and jar /path/to/jar"
+          }
+        ]
+      }
+    }
+  },
+  "variables": {
+    "cluster": {
+      "default": {
+        "node_type_id": "Standard_DS3_v3",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      },
+      "description": "A cluster definition",
+      "type": "complex",
+      "value": {
+        "node_type_id": "Standard_DS3_v3",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      }
+    },
+    "complexvar": {
+      "default": {
+        "key1": "1",
+        "key2": "2",
+        "key3": "3"
+      },
+      "description": "A complex variable",
+      "type": "complex",
+      "value": {
+        "key1": "1",
+        "key2": "2",
+        "key3": "3"
+      }
+    },
+    "libraries": {
+      "default": [
+        {
+          "jar": "/path/to/jar"
+        },
+        {
+          "egg": "/path/to/egg"
+        },
+        {
+          "whl": "/path/to/whl"
+        }
+      ],
+      "description": "A libraries definition",
+      "type": "complex",
+      "value": [
+        {
+          "jar": "/path/to/jar"
+        },
+        {
+          "egg": "/path/to/egg"
+        },
+        {
+          "whl": "/path/to/whl"
+        }
+      ]
+    },
+    "node_type": {
+      "default": "Standard_DS3_v3",
+      "value": "Standard_DS3_v3"
+    }
+  }
+}
diff --git a/acceptance/bundle/variables/complex/script b/acceptance/bundle/variables/complex/script
new file mode 100644
index 000000000..9ee0ab02a
--- /dev/null
+++ b/acceptance/bundle/variables/complex/script
@@ -0,0 +1,4 @@
+$CLI bundle validate -o json | jq '{resources,variables}' > out.default.json
+
+# spark.random and policy_id should be empty in this target:
+$CLI bundle validate -o json -t dev | jq '{resources,variables}' > out.dev.json
diff --git a/bundle/tests/variables/complex_multiple_files/databricks.yml b/acceptance/bundle/variables/complex_multiple_files/databricks.yml
similarity index 100%
rename from bundle/tests/variables/complex_multiple_files/databricks.yml
rename to acceptance/bundle/variables/complex_multiple_files/databricks.yml
diff --git a/acceptance/bundle/variables/complex_multiple_files/output.txt b/acceptance/bundle/variables/complex_multiple_files/output.txt
new file mode 100644
index 000000000..e87b8df11
--- /dev/null
+++ b/acceptance/bundle/variables/complex_multiple_files/output.txt
@@ -0,0 +1,159 @@
+{
+  "resources": {
+    "jobs": {
+      "my_job": {
+        "deployment": {
+          "kind": "BUNDLE",
+          "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/complex-variables-multiple-files/dev/state/metadata.json"
+        },
+        "edit_mode": "UI_LOCKED",
+        "format": "MULTI_TASK",
+        "job_clusters": [
+          {
+            "job_cluster_key": "key1",
+            "new_cluster": {
+              "node_type_id": "Standard_DS3_v2",
+              "num_workers": 4,
+              "spark_conf": {
+                "spark.databricks.delta.retentionDurationCheck.enabled": "false",
+                "spark.speculation": "false"
+              },
+              "spark_version": "14.2.x-scala2.11"
+            }
+          },
+          {
+            "job_cluster_key": "key2",
+            "new_cluster": {
+              "node_type_id": "Standard_DS3_v2",
+              "num_workers": 4,
+              "spark_conf": {
+                "spark.databricks.delta.retentionDurationCheck.enabled": "false",
+                "spark.speculation": "false"
+              },
+              "spark_version": "14.2.x-scala2.11"
+            }
+          },
+          {
+            "job_cluster_key": "key3",
+            "new_cluster": {
+              "node_type_id": "Standard_DS3_v2",
+              "num_workers": 4,
+              "spark_conf": {
+                "spark.databricks.delta.retentionDurationCheck.enabled": "false",
+                "spark.speculation": "false"
+              },
+              "spark_version": "14.2.x-scala2.11"
+            }
+          },
+          {
+            "job_cluster_key": "key4",
+            "new_cluster": {
+              "node_type_id": "Standard_DS3_v2",
+              "num_workers": 4,
+              "spark_conf": {
+                "spark.databricks.delta.retentionDurationCheck.enabled": "false",
+                "spark.speculation": "false"
+              },
+              "spark_version": "14.2.x-scala2.11"
+            }
+          }
+        ],
+        "permissions": [],
+        "queue": {
+          "enabled": true
+        },
+        "tags": {}
+      }
+    }
+  },
+  "variables": {
+    "cluster1": {
+      "default": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      },
+      "description": "A cluster definition",
+      "type": "complex",
+      "value": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      }
+    },
+    "cluster2": {
+      "default": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      },
+      "description": "A cluster definition",
+      "type": "complex",
+      "value": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      }
+    },
+    "cluster3": {
+      "default": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      },
+      "description": "A cluster definition",
+      "type": "complex",
+      "value": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      }
+    },
+    "cluster4": {
+      "default": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      },
+      "description": "A cluster definition",
+      "type": "complex",
+      "value": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      }
+    }
+  }
+}
diff --git a/acceptance/bundle/variables/complex_multiple_files/script b/acceptance/bundle/variables/complex_multiple_files/script
new file mode 100644
index 000000000..24f1d58d5
--- /dev/null
+++ b/acceptance/bundle/variables/complex_multiple_files/script
@@ -0,0 +1 @@
+$CLI bundle validate -t dev -o json | jq '{resources, variables}'
diff --git a/bundle/tests/variables/complex_multiple_files/variables/clusters.yml b/acceptance/bundle/variables/complex_multiple_files/variables/clusters.yml
similarity index 100%
rename from bundle/tests/variables/complex_multiple_files/variables/clusters.yml
rename to acceptance/bundle/variables/complex_multiple_files/variables/clusters.yml
diff --git a/bundle/tests/variables/empty/databricks.yml b/acceptance/bundle/variables/empty/databricks.yml
similarity index 100%
rename from bundle/tests/variables/empty/databricks.yml
rename to acceptance/bundle/variables/empty/databricks.yml
diff --git a/acceptance/bundle/variables/empty/output.txt b/acceptance/bundle/variables/empty/output.txt
new file mode 100644
index 000000000..c3f0af130
--- /dev/null
+++ b/acceptance/bundle/variables/empty/output.txt
@@ -0,0 +1,11 @@
+Error: no value assigned to required variable a. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_a environment variable
+
+Name: empty${var.a}
+Target: default
+Workspace:
+  User: tester@databricks.com
+  Path: /Workspace/Users/tester@databricks.com/.bundle/empty${var.a}/default
+
+Found 1 error
+
+Exit code: 1
diff --git a/acceptance/bundle/variables/empty/script b/acceptance/bundle/variables/empty/script
new file mode 100644
index 000000000..72555b332
--- /dev/null
+++ b/acceptance/bundle/variables/empty/script
@@ -0,0 +1 @@
+$CLI bundle validate
diff --git a/bundle/tests/variables/env_overrides/databricks.yml b/acceptance/bundle/variables/env_overrides/databricks.yml
similarity index 100%
rename from bundle/tests/variables/env_overrides/databricks.yml
rename to acceptance/bundle/variables/env_overrides/databricks.yml
diff --git a/acceptance/bundle/variables/env_overrides/output.txt b/acceptance/bundle/variables/env_overrides/output.txt
new file mode 100644
index 000000000..e8fb99938
--- /dev/null
+++ b/acceptance/bundle/variables/env_overrides/output.txt
@@ -0,0 +1,40 @@
+
+>>> $CLI bundle validate -t env-with-single-variable-override -o json
+"default-a dev-b"
+
+>>> $CLI bundle validate -t env-with-two-variable-overrides -o json
+"prod-a prod-b"
+
+>>> BUNDLE_VAR_b=env-var-b $CLI bundle validate -t env-with-two-variable-overrides -o json
+"prod-a env-var-b"
+
+>>> errcode $CLI bundle validate -t env-missing-a-required-variable-assignment
+Error: no value assigned to required variable b. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_b environment variable
+
+Name: test bundle
+Target: env-missing-a-required-variable-assignment
+Workspace:
+  User: tester@databricks.com
+  Path: /Workspace/Users/tester@databricks.com/.bundle/test bundle/env-missing-a-required-variable-assignment
+
+Found 1 error
+
+Exit code: 1
+
+>>> errcode $CLI bundle validate -t env-using-an-undefined-variable
+Error: variable c is not defined but is assigned a value
+
+Name: test bundle
+
+Found 1 error
+
+Exit code: 1
+
+>>> $CLI bundle validate -t env-overrides-lookup -o json
+{
+  "a": "default-a",
+  "b": "prod-b",
+  "d": "4321",
+  "e": "1234",
+  "f": "9876"
+}
diff --git a/acceptance/bundle/variables/env_overrides/script b/acceptance/bundle/variables/env_overrides/script
new file mode 100644
index 000000000..30919fd8a
--- /dev/null
+++ b/acceptance/bundle/variables/env_overrides/script
@@ -0,0 +1,6 @@
+trace $CLI bundle validate -t env-with-single-variable-override -o json | jq .workspace.profile
+trace $CLI bundle validate -t env-with-two-variable-overrides -o json | jq .workspace.profile
+trace BUNDLE_VAR_b=env-var-b $CLI bundle validate -t env-with-two-variable-overrides -o json | jq .workspace.profile
+trace errcode $CLI bundle validate -t env-missing-a-required-variable-assignment
+trace errcode $CLI bundle validate -t env-using-an-undefined-variable
+trace $CLI bundle validate -t env-overrides-lookup -o json | jq '.variables | map_values(.value)'
diff --git a/bundle/tests/variables/vanilla/databricks.yml b/acceptance/bundle/variables/vanilla/databricks.yml
similarity index 100%
rename from bundle/tests/variables/vanilla/databricks.yml
rename to acceptance/bundle/variables/vanilla/databricks.yml
diff --git a/acceptance/bundle/variables/vanilla/output.txt b/acceptance/bundle/variables/vanilla/output.txt
new file mode 100644
index 000000000..69b358a3f
--- /dev/null
+++ b/acceptance/bundle/variables/vanilla/output.txt
@@ -0,0 +1,16 @@
+
+>>> BUNDLE_VAR_b=def $CLI bundle validate -o json
+"abc def"
+
+>>> errcode $CLI bundle validate
+Error: no value assigned to required variable b. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_b environment variable
+
+Name: ${var.a} ${var.b}
+Target: default
+Workspace:
+  User: tester@databricks.com
+  Path: /Workspace/Users/tester@databricks.com/.bundle/${var.a} ${var.b}/default
+
+Found 1 error
+
+Exit code: 1
diff --git a/acceptance/bundle/variables/vanilla/script b/acceptance/bundle/variables/vanilla/script
new file mode 100644
index 000000000..10da3183d
--- /dev/null
+++ b/acceptance/bundle/variables/vanilla/script
@@ -0,0 +1,2 @@
+trace BUNDLE_VAR_b=def $CLI bundle validate -o json | jq .bundle.name
+trace errcode $CLI bundle validate
diff --git a/bundle/tests/variables/variable_overrides_in_target/databricks.yml b/acceptance/bundle/variables/variable_overrides_in_target/databricks.yml
similarity index 100%
rename from bundle/tests/variables/variable_overrides_in_target/databricks.yml
rename to acceptance/bundle/variables/variable_overrides_in_target/databricks.yml
diff --git a/acceptance/bundle/variables/variable_overrides_in_target/output.txt b/acceptance/bundle/variables/variable_overrides_in_target/output.txt
new file mode 100644
index 000000000..de193f5b6
--- /dev/null
+++ b/acceptance/bundle/variables/variable_overrides_in_target/output.txt
@@ -0,0 +1,84 @@
+
+>>> $CLI bundle validate -o json -t use-default-variable-values
+{
+  "pipelines": {
+    "my_pipeline": {
+      "clusters": [
+        {
+          "label": "default",
+          "num_workers": 42
+        }
+      ],
+      "continuous": true,
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/foobar/use-default-variable-values/state/metadata.json"
+      },
+      "name": "a_string",
+      "permissions": []
+    }
+  }
+}
+
+>>> $CLI bundle validate -o json -t override-string-variable
+{
+  "pipelines": {
+    "my_pipeline": {
+      "clusters": [
+        {
+          "label": "default",
+          "num_workers": 42
+        }
+      ],
+      "continuous": true,
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/foobar/override-string-variable/state/metadata.json"
+      },
+      "name": "overridden_string",
+      "permissions": []
+    }
+  }
+}
+
+>>> $CLI bundle validate -o json -t override-int-variable
+{
+  "pipelines": {
+    "my_pipeline": {
+      "clusters": [
+        {
+          "label": "default",
+          "num_workers": 43
+        }
+      ],
+      "continuous": true,
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/foobar/override-int-variable/state/metadata.json"
+      },
+      "name": "a_string",
+      "permissions": []
+    }
+  }
+}
+
+>>> $CLI bundle validate -o json -t override-both-bool-and-string-variables
+{
+  "pipelines": {
+    "my_pipeline": {
+      "clusters": [
+        {
+          "label": "default",
+          "num_workers": 42
+        }
+      ],
+      "continuous": false,
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/foobar/override-both-bool-and-string-variables/state/metadata.json"
+      },
+      "name": "overridden_string",
+      "permissions": []
+    }
+  }
+}
diff --git a/acceptance/bundle/variables/variable_overrides_in_target/script b/acceptance/bundle/variables/variable_overrides_in_target/script
new file mode 100644
index 000000000..686b3102a
--- /dev/null
+++ b/acceptance/bundle/variables/variable_overrides_in_target/script
@@ -0,0 +1,4 @@
+trace $CLI bundle validate -o json -t use-default-variable-values | jq .resources
+trace $CLI bundle validate -o json -t override-string-variable | jq .resources
+trace $CLI bundle validate -o json -t override-int-variable | jq .resources
+trace $CLI bundle validate -o json -t override-both-bool-and-string-variables | jq .resources
diff --git a/bundle/tests/variables/without_definition/databricks.yml b/acceptance/bundle/variables/without_definition/databricks.yml
similarity index 53%
rename from bundle/tests/variables/without_definition/databricks.yml
rename to acceptance/bundle/variables/without_definition/databricks.yml
index 68227b683..c26a85f56 100644
--- a/bundle/tests/variables/without_definition/databricks.yml
+++ b/acceptance/bundle/variables/without_definition/databricks.yml
@@ -1,3 +1,5 @@
+bundle:
+  name: x
 variables:
   a:
   b:
diff --git a/acceptance/bundle/variables/without_definition/output.txt b/acceptance/bundle/variables/without_definition/output.txt
new file mode 100644
index 000000000..4dd1e6609
--- /dev/null
+++ b/acceptance/bundle/variables/without_definition/output.txt
@@ -0,0 +1,4 @@
+{
+  "a": "foo",
+  "b": "bar"
+}
diff --git a/acceptance/bundle/variables/without_definition/script b/acceptance/bundle/variables/without_definition/script
new file mode 100644
index 000000000..49b9b5448
--- /dev/null
+++ b/acceptance/bundle/variables/without_definition/script
@@ -0,0 +1 @@
+BUNDLE_VAR_a=foo BUNDLE_VAR_b=bar $CLI bundle validate -o json | jq '.variables | map_values(.value)'
diff --git a/acceptance/help/output.txt b/acceptance/help/output.txt
new file mode 100644
index 000000000..ed4a88ce6
--- /dev/null
+++ b/acceptance/help/output.txt
@@ -0,0 +1,143 @@
+Databricks CLI
+
+Usage:
+  databricks [command]
+
+Databricks Workspace
+  fs                                     Filesystem related commands
+  git-credentials                        Registers personal access token for Databricks to do operations on behalf of the user.
+  repos                                  The Repos API allows users to manage their git repos.
+  secrets                                The Secrets API allows you to manage secrets, secret scopes, and access permissions.
+  workspace                              The Workspace API allows you to list, import, export, and delete notebooks and folders.
+
+Compute
+  cluster-policies                       You can use cluster policies to control users' ability to configure clusters based on a set of rules.
+  clusters                               The Clusters API allows you to create, start, edit, list, terminate, and delete clusters.
+  global-init-scripts                    The Global Init Scripts API enables Workspace administrators to configure global initialization scripts for their workspace.
+  instance-pools                         Instance Pools API are used to create, edit, delete and list instance pools by using ready-to-use cloud instances which reduces a cluster start and auto-scaling times.
+  instance-profiles                      The Instance Profiles API allows admins to add, list, and remove instance profiles that users can launch clusters with.
+  libraries                              The Libraries API allows you to install and uninstall libraries and get the status of libraries on a cluster.
+  policy-compliance-for-clusters         The policy compliance APIs allow you to view and manage the policy compliance status of clusters in your workspace.
+  policy-families                        View available policy families.
+
+Workflows
+  jobs                                   The Jobs API allows you to create, edit, and delete jobs.
+  policy-compliance-for-jobs             The compliance APIs allow you to view and manage the policy compliance status of jobs in your workspace.
+
+Delta Live Tables
+  pipelines                              The Delta Live Tables API allows you to create, edit, delete, start, and view details about pipelines.
+
+Machine Learning
+  experiments                            Experiments are the primary unit of organization in MLflow; all MLflow runs belong to an experiment.
+  model-registry                         Note: This API reference documents APIs for the Workspace Model Registry.
+
+Real-time Serving
+  serving-endpoints                      The Serving Endpoints API allows you to create, update, and delete model serving endpoints.
+
+Identity and Access Management
+  current-user                           This API allows retrieving information about currently authenticated user or service principal.
+  groups                                 Groups simplify identity management, making it easier to assign access to Databricks workspace, data, and other securable objects.
+  permissions                            Permissions API are used to create read, write, edit, update and manage access for various users on different objects and endpoints.
+  service-principals                     Identities for use with jobs, automated tools, and systems such as scripts, apps, and CI/CD platforms.
+  users                                  User identities recognized by Databricks and represented by email addresses.
+
+Databricks SQL
+  alerts                                 The alerts API can be used to perform CRUD operations on alerts.
+  alerts-legacy                          The alerts API can be used to perform CRUD operations on alerts.
+  dashboards                             In general, there is little need to modify dashboards using the API.
+  data-sources                           This API is provided to assist you in making new query objects.
+  queries                                The queries API can be used to perform CRUD operations on queries.
+  queries-legacy                         These endpoints are used for CRUD operations on query definitions.
+  query-history                          A service responsible for storing and retrieving the list of queries run against SQL endpoints and serverless compute.
+  warehouses                             A SQL warehouse is a compute resource that lets you run SQL commands on data objects within Databricks SQL.
+
+Unity Catalog
+  artifact-allowlists                    In Databricks Runtime 13.3 and above, you can add libraries and init scripts to the allowlist in UC so that users can leverage these artifacts on compute configured with shared access mode.
+  catalogs                               A catalog is the first layer of Unity Catalog’s three-level namespace.
+  connections                            Connections allow for creating a connection to an external data source.
+  credentials                            A credential represents an authentication and authorization mechanism for accessing services on your cloud tenant.
+  external-locations                     An external location is an object that combines a cloud storage path with a storage credential that authorizes access to the cloud storage path.
+  functions                              Functions implement User-Defined Functions (UDFs) in Unity Catalog.
+  grants                                 In Unity Catalog, data is secure by default.
+  metastores                             A metastore is the top-level container of objects in Unity Catalog.
+  model-versions                         Databricks provides a hosted version of MLflow Model Registry in Unity Catalog.
+  online-tables                          Online tables provide lower latency and higher QPS access to data from Delta tables.
+  quality-monitors                       A monitor computes and monitors data or model quality metrics for a table over time.
+  registered-models                      Databricks provides a hosted version of MLflow Model Registry in Unity Catalog.
+  resource-quotas                        Unity Catalog enforces resource quotas on all securable objects, which limits the number of resources that can be created.
+  schemas                                A schema (also called a database) is the second layer of Unity Catalog’s three-level namespace.
+  storage-credentials                    A storage credential represents an authentication and authorization mechanism for accessing data stored on your cloud tenant.
+  system-schemas                         A system schema is a schema that lives within the system catalog.
+  table-constraints                      Primary key and foreign key constraints encode relationships between fields in tables.
+  tables                                 A table resides in the third layer of Unity Catalog’s three-level namespace.
+  temporary-table-credentials            Temporary Table Credentials refer to short-lived, downscoped credentials used to access cloud storage locationswhere table data is stored in Databricks.
+  volumes                                Volumes are a Unity Catalog (UC) capability for accessing, storing, governing, organizing and processing files.
+  workspace-bindings                     A securable in Databricks can be configured as __OPEN__ or __ISOLATED__.
+
+Delta Sharing
+  providers                              A data provider is an object representing the organization in the real world who shares the data.
+  recipient-activation                   The Recipient Activation API is only applicable in the open sharing model where the recipient object has the authentication type of TOKEN.
+  recipients                             A recipient is an object you create using :method:recipients/create to represent an organization which you want to allow access shares.
+  shares                                 A share is a container instantiated with :method:shares/create.
+
+Settings
+  ip-access-lists                        IP Access List enables admins to configure IP access lists.
+  notification-destinations              The notification destinations API lets you programmatically manage a workspace's notification destinations.
+  settings                               Workspace Settings API allows users to manage settings at the workspace level.
+  token-management                       Enables administrators to get all tokens and delete tokens for other users.
+  tokens                                 The Token API allows you to create, list, and revoke tokens that can be used to authenticate and access Databricks REST APIs.
+  workspace-conf                         This API allows updating known workspace settings for advanced users.
+
+Developer Tools
+  bundle                                 Databricks Asset Bundles let you express data/AI/analytics projects as code.
+  sync                                   Synchronize a local directory to a workspace directory
+
+Vector Search
+  vector-search-endpoints                **Endpoint**: Represents the compute resources to host vector search indexes.
+  vector-search-indexes                  **Index**: An efficient representation of your embedding vectors that supports real-time and efficient approximate nearest neighbor (ANN) search queries.
+
+Dashboards
+  lakeview                               These APIs provide specific management operations for Lakeview dashboards.
+
+Marketplace
+  consumer-fulfillments                  Fulfillments are entities that allow consumers to preview installations.
+  consumer-installations                 Installations are entities that allow consumers to interact with Databricks Marketplace listings.
+  consumer-listings                      Listings are the core entities in the Marketplace.
+  consumer-personalization-requests      Personalization Requests allow customers to interact with the individualized Marketplace listing flow.
+  consumer-providers                     Providers are the entities that publish listings to the Marketplace.
+  provider-exchange-filters              Marketplace exchanges filters curate which groups can access an exchange.
+  provider-exchanges                     Marketplace exchanges allow providers to share their listings with a curated set of customers.
+  provider-files                         Marketplace offers a set of file APIs for various purposes such as preview notebooks and provider icons.
+  provider-listings                      Listings are the core entities in the Marketplace.
+  provider-personalization-requests      Personalization requests are an alternate to instantly available listings.
+  provider-provider-analytics-dashboards Manage templated analytics solution for providers.
+  provider-providers                     Providers are entities that manage assets in Marketplace.
+
+Apps
+  apps                                   Apps run directly on a customer’s Databricks instance, integrate with their data, use and extend Databricks services, and enable users to interact through single sign-on.
+  apps                                   Apps run directly on a customer’s Databricks instance, integrate with their data, use and extend Databricks services, and enable users to interact through single sign-on.
+
+Clean Rooms
+  clean-room-assets                      Clean room assets are data and code objects — Tables, volumes, and notebooks that are shared with the clean room.
+  clean-room-task-runs                   Clean room task runs are the executions of notebooks in a clean room.
+  clean-rooms                            A clean room uses Delta Sharing and serverless compute to provide a secure and privacy-protecting environment where multiple parties can work together on sensitive enterprise data without direct access to each other’s data.
+
+Additional Commands:
+  account                                Databricks Account Commands
+  api                                    Perform Databricks API call
+  auth                                   Authentication related commands
+  completion                             Generate the autocompletion script for the specified shell
+  configure                              Configure authentication
+  help                                   Help about any command
+  labs                                   Manage Databricks Labs installations
+  version                                Retrieve information about the current version of this CLI
+
+Flags:
+      --debug            enable debug logging
+  -h, --help             help for databricks
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+  -v, --version          version for databricks
+
+Use "databricks [command] --help" for more information about a command.
diff --git a/acceptance/help/script b/acceptance/help/script
new file mode 100644
index 000000000..5fa569470
--- /dev/null
+++ b/acceptance/help/script
@@ -0,0 +1 @@
+$CLI
diff --git a/acceptance/script.cleanup b/acceptance/script.cleanup
new file mode 100644
index 000000000..3c3e29ebc
--- /dev/null
+++ b/acceptance/script.cleanup
@@ -0,0 +1 @@
+rm -fr .databricks .gitignore
diff --git a/acceptance/script.prepare b/acceptance/script.prepare
new file mode 100644
index 000000000..3f1bb2acc
--- /dev/null
+++ b/acceptance/script.prepare
@@ -0,0 +1,36 @@
+# Prevent CLI from downloading terraform in each test:
+export DATABRICKS_TF_EXEC_PATH=/tmp/
+
+errcode() {
+    # Temporarily disable 'set -e' to prevent the script from exiting on error
+    set +e
+    # Execute the provided command with all arguments
+    "$@"
+    local exit_code=$?
+    # Re-enable 'set -e' if it was previously set
+    set -e
+    >&2 printf "\nExit code: $exit_code\n"
+}
+
+trace() {
+    >&2 printf "\n>>> %s\n" "$*"
+
+    if [[ "$1" == *"="* ]]; then
+        # If the first argument contains '=', collect all env vars
+        local env_vars=()
+        while [[ "$1" == *"="* ]]; do
+            env_vars+=("$1")
+            shift
+        done
+        # Export environment variables in a subshell and execute the command
+        (
+            export "${env_vars[@]}"
+            "$@"
+        )
+    else
+        # Execute the command normally
+        "$@"
+    fi
+
+    return $?
+}
diff --git a/acceptance/server_test.go b/acceptance/server_test.go
new file mode 100644
index 000000000..7b21e198f
--- /dev/null
+++ b/acceptance/server_test.go
@@ -0,0 +1,129 @@
+package acceptance_test
+
+import (
+	"encoding/json"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/service/compute"
+	"github.com/databricks/databricks-sdk-go/service/iam"
+	"github.com/databricks/databricks-sdk-go/service/workspace"
+)
+
+type TestServer struct {
+	*httptest.Server
+	Mux  *http.ServeMux
+	Port int
+}
+
+type HandlerFunc func(r *http.Request) (any, error)
+
+func NewTestServer() *TestServer {
+	mux := http.NewServeMux()
+	server := httptest.NewServer(mux)
+	port := server.Listener.Addr().(*net.TCPAddr).Port
+
+	return &TestServer{
+		Server: server,
+		Mux:    mux,
+		Port:   port,
+	}
+}
+
+func (s *TestServer) Handle(pattern string, handler HandlerFunc) {
+	s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) {
+		resp, err := handler(r)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+
+		var respBytes []byte
+
+		respString, ok := resp.(string)
+		if ok {
+			respBytes = []byte(respString)
+		} else {
+			respBytes, err = json.MarshalIndent(resp, "", "    ")
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
+		}
+
+		if _, err := w.Write(respBytes); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	})
+}
+
+func StartServer(t *testing.T) *TestServer {
+	server := NewTestServer()
+	t.Cleanup(func() {
+		server.Close()
+	})
+	return server
+}
+
+func AddHandlers(server *TestServer) {
+	server.Handle("/api/2.0/policies/clusters/list", func(r *http.Request) (any, error) {
+		return compute.ListPoliciesResponse{
+			Policies: []compute.Policy{
+				{
+					PolicyId: "5678",
+					Name:     "wrong-cluster-policy",
+				},
+				{
+					PolicyId: "9876",
+					Name:     "some-test-cluster-policy",
+				},
+			},
+		}, nil
+	})
+
+	server.Handle("/api/2.0/instance-pools/list", func(r *http.Request) (any, error) {
+		return compute.ListInstancePools{
+			InstancePools: []compute.InstancePoolAndStats{
+				{
+					InstancePoolName: "some-test-instance-pool",
+					InstancePoolId:   "1234",
+				},
+			},
+		}, nil
+	})
+
+	server.Handle("/api/2.1/clusters/list", func(r *http.Request) (any, error) {
+		return compute.ListClustersResponse{
+			Clusters: []compute.ClusterDetails{
+				{
+					ClusterName: "some-test-cluster",
+					ClusterId:   "4321",
+				},
+				{
+					ClusterName: "some-other-cluster",
+					ClusterId:   "9876",
+				},
+			},
+		}, nil
+	})
+
+	server.Handle("/api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, error) {
+		return iam.User{
+			UserName: "tester@databricks.com",
+		}, nil
+	})
+
+	server.Handle("/api/2.0/workspace/get-status", func(r *http.Request) (any, error) {
+		return workspace.ObjectInfo{
+			ObjectId:   1001,
+			ObjectType: "DIRECTORY",
+			Path:       "",
+			ResourceId: "1001",
+		}, nil
+	})
+}
diff --git a/bundle/tests/clusters_test.go b/bundle/tests/clusters_test.go
deleted file mode 100644
index def8a2a31..000000000
--- a/bundle/tests/clusters_test.go
+++ /dev/null
@@ -1,36 +0,0 @@
-package config_tests
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestClusters(t *testing.T) {
-	b := load(t, "./clusters")
-	assert.Equal(t, "clusters", b.Config.Bundle.Name)
-
-	cluster := b.Config.Resources.Clusters["foo"]
-	assert.Equal(t, "foo", cluster.ClusterName)
-	assert.Equal(t, "13.3.x-scala2.12", cluster.SparkVersion)
-	assert.Equal(t, "i3.xlarge", cluster.NodeTypeId)
-	assert.Equal(t, 2, cluster.NumWorkers)
-	assert.Equal(t, "2g", cluster.SparkConf["spark.executor.memory"])
-	assert.Equal(t, 2, cluster.Autoscale.MinWorkers)
-	assert.Equal(t, 7, cluster.Autoscale.MaxWorkers)
-}
-
-func TestClustersOverride(t *testing.T) {
-	b := loadTarget(t, "./clusters", "development")
-	assert.Equal(t, "clusters", b.Config.Bundle.Name)
-
-	cluster := b.Config.Resources.Clusters["foo"]
-	assert.Equal(t, "foo-override", cluster.ClusterName)
-	assert.Equal(t, "15.2.x-scala2.12", cluster.SparkVersion)
-	assert.Equal(t, "m5.xlarge", cluster.NodeTypeId)
-	assert.Equal(t, 3, cluster.NumWorkers)
-	assert.Equal(t, "4g", cluster.SparkConf["spark.executor.memory"])
-	assert.Equal(t, "4g", cluster.SparkConf["spark.executor.memory2"])
-	assert.Equal(t, 1, cluster.Autoscale.MinWorkers)
-	assert.Equal(t, 3, cluster.Autoscale.MaxWorkers)
-}
diff --git a/bundle/tests/complex_variables_test.go b/bundle/tests/complex_variables_test.go
deleted file mode 100644
index d72b5f157..000000000
--- a/bundle/tests/complex_variables_test.go
+++ /dev/null
@@ -1,108 +0,0 @@
-package config_tests
-
-import (
-	"context"
-	"testing"
-
-	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/bundle/config/mutator"
-	"github.com/databricks/databricks-sdk-go/service/compute"
-	"github.com/stretchr/testify/require"
-)
-
-func TestComplexVariables(t *testing.T) {
-	b, diags := loadTargetWithDiags("variables/complex", "default")
-	require.Empty(t, diags)
-
-	diags = bundle.Apply(context.Background(), b, bundle.Seq(
-		mutator.SetVariables(),
-		mutator.ResolveVariableReferencesInComplexVariables(),
-		mutator.ResolveVariableReferences(
-			"variables",
-		),
-	))
-	require.NoError(t, diags.Error())
-
-	require.Equal(t, "13.2.x-scala2.11", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkVersion)
-	require.Equal(t, "Standard_DS3_v2", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.NodeTypeId)
-	require.Equal(t, "some-policy-id", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.PolicyId)
-	require.Equal(t, 2, b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.NumWorkers)
-	require.Equal(t, "true", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkConf["spark.speculation"])
-	require.Equal(t, "true", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkConf["spark.random"])
-
-	require.Len(t, b.Config.Resources.Jobs["my_job"].Tasks[0].Libraries, 3)
-	require.Contains(t, b.Config.Resources.Jobs["my_job"].Tasks[0].Libraries, compute.Library{
-		Jar: "/path/to/jar",
-	})
-	require.Contains(t, b.Config.Resources.Jobs["my_job"].Tasks[0].Libraries, compute.Library{
-		Egg: "/path/to/egg",
-	})
-	require.Contains(t, b.Config.Resources.Jobs["my_job"].Tasks[0].Libraries, compute.Library{
-		Whl: "/path/to/whl",
-	})
-
-	require.Equal(t, "task with spark version 13.2.x-scala2.11 and jar /path/to/jar", b.Config.Resources.Jobs["my_job"].Tasks[0].TaskKey)
-}
-
-func TestComplexVariablesOverride(t *testing.T) {
-	b, diags := loadTargetWithDiags("variables/complex", "dev")
-	require.Empty(t, diags)
-
-	diags = bundle.Apply(context.Background(), b, bundle.Seq(
-		mutator.SetVariables(),
-		mutator.ResolveVariableReferencesInComplexVariables(),
-		mutator.ResolveVariableReferences(
-			"variables",
-		),
-	))
-	require.NoError(t, diags.Error())
-
-	require.Equal(t, "14.2.x-scala2.11", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkVersion)
-	require.Equal(t, "Standard_DS3_v3", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.NodeTypeId)
-	require.Equal(t, 4, b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.NumWorkers)
-	require.Equal(t, "false", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkConf["spark.speculation"])
-
-	// Making sure the variable is overriden and not merged / extended
-	// These properties are set in the default target but not set in override target
-	// So they should be empty
-	require.Equal(t, "", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkConf["spark.random"])
-	require.Equal(t, "", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.PolicyId)
-}
-
-func TestComplexVariablesOverrideWithMultipleFiles(t *testing.T) {
-	b, diags := loadTargetWithDiags("variables/complex_multiple_files", "dev")
-	require.Empty(t, diags)
-
-	diags = bundle.Apply(context.Background(), b, bundle.Seq(
-		mutator.SetVariables(),
-		mutator.ResolveVariableReferencesInComplexVariables(),
-		mutator.ResolveVariableReferences(
-			"variables",
-		),
-	))
-	require.NoError(t, diags.Error())
-	for _, cluster := range b.Config.Resources.Jobs["my_job"].JobClusters {
-		require.Equalf(t, "14.2.x-scala2.11", cluster.NewCluster.SparkVersion, "cluster: %v", cluster.JobClusterKey)
-		require.Equalf(t, "Standard_DS3_v2", cluster.NewCluster.NodeTypeId, "cluster: %v", cluster.JobClusterKey)
-		require.Equalf(t, 4, cluster.NewCluster.NumWorkers, "cluster: %v", cluster.JobClusterKey)
-		require.Equalf(t, "false", cluster.NewCluster.SparkConf["spark.speculation"], "cluster: %v", cluster.JobClusterKey)
-	}
-}
-
-func TestComplexVariablesOverrideWithFullSyntax(t *testing.T) {
-	b, diags := loadTargetWithDiags("variables/complex", "dev")
-	require.Empty(t, diags)
-
-	diags = bundle.Apply(context.Background(), b, bundle.Seq(
-		mutator.SetVariables(),
-		mutator.ResolveVariableReferencesInComplexVariables(),
-		mutator.ResolveVariableReferences(
-			"variables",
-		),
-	))
-	require.NoError(t, diags.Error())
-	require.Empty(t, diags)
-
-	complexvar := b.Config.Variables["complexvar"].Value
-	require.Equal(t, map[string]any{"key1": "1", "key2": "2", "key3": "3"}, complexvar)
-}
diff --git a/bundle/tests/override_job_cluster_test.go b/bundle/tests/override_job_cluster_test.go
deleted file mode 100644
index 1393e03e5..000000000
--- a/bundle/tests/override_job_cluster_test.go
+++ /dev/null
@@ -1,29 +0,0 @@
-package config_tests
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestOverrideJobClusterDev(t *testing.T) {
-	b := loadTarget(t, "./override_job_cluster", "development")
-	assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name)
-	assert.Len(t, b.Config.Resources.Jobs["foo"].JobClusters, 1)
-
-	c := b.Config.Resources.Jobs["foo"].JobClusters[0]
-	assert.Equal(t, "13.3.x-scala2.12", c.NewCluster.SparkVersion)
-	assert.Equal(t, "i3.xlarge", c.NewCluster.NodeTypeId)
-	assert.Equal(t, 1, c.NewCluster.NumWorkers)
-}
-
-func TestOverrideJobClusterStaging(t *testing.T) {
-	b := loadTarget(t, "./override_job_cluster", "staging")
-	assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name)
-	assert.Len(t, b.Config.Resources.Jobs["foo"].JobClusters, 1)
-
-	c := b.Config.Resources.Jobs["foo"].JobClusters[0]
-	assert.Equal(t, "13.3.x-scala2.12", c.NewCluster.SparkVersion)
-	assert.Equal(t, "i3.2xlarge", c.NewCluster.NodeTypeId)
-	assert.Equal(t, 4, c.NewCluster.NumWorkers)
-}
diff --git a/bundle/tests/override_job_tasks_test.go b/bundle/tests/override_job_tasks_test.go
deleted file mode 100644
index 85463e17c..000000000
--- a/bundle/tests/override_job_tasks_test.go
+++ /dev/null
@@ -1,39 +0,0 @@
-package config_tests
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestOverrideTasksDev(t *testing.T) {
-	b := loadTarget(t, "./override_job_tasks", "development")
-	assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name)
-	assert.Len(t, b.Config.Resources.Jobs["foo"].Tasks, 2)
-
-	tasks := b.Config.Resources.Jobs["foo"].Tasks
-	assert.Equal(t, "key1", tasks[0].TaskKey)
-	assert.Equal(t, "i3.xlarge", tasks[0].NewCluster.NodeTypeId)
-	assert.Equal(t, 1, tasks[0].NewCluster.NumWorkers)
-	assert.Equal(t, "./test1.py", tasks[0].SparkPythonTask.PythonFile)
-
-	assert.Equal(t, "key2", tasks[1].TaskKey)
-	assert.Equal(t, "13.3.x-scala2.12", tasks[1].NewCluster.SparkVersion)
-	assert.Equal(t, "./test2.py", tasks[1].SparkPythonTask.PythonFile)
-}
-
-func TestOverrideTasksStaging(t *testing.T) {
-	b := loadTarget(t, "./override_job_tasks", "staging")
-	assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name)
-	assert.Len(t, b.Config.Resources.Jobs["foo"].Tasks, 2)
-
-	tasks := b.Config.Resources.Jobs["foo"].Tasks
-	assert.Equal(t, "key1", tasks[0].TaskKey)
-	assert.Equal(t, "13.3.x-scala2.12", tasks[0].NewCluster.SparkVersion)
-	assert.Equal(t, "./test1.py", tasks[0].SparkPythonTask.PythonFile)
-
-	assert.Equal(t, "key2", tasks[1].TaskKey)
-	assert.Equal(t, "i3.2xlarge", tasks[1].NewCluster.NodeTypeId)
-	assert.Equal(t, 4, tasks[1].NewCluster.NumWorkers)
-	assert.Equal(t, "./test3.py", tasks[1].SparkPythonTask.PythonFile)
-}
diff --git a/bundle/tests/override_pipeline_cluster_test.go b/bundle/tests/override_pipeline_cluster_test.go
deleted file mode 100644
index 591fe423d..000000000
--- a/bundle/tests/override_pipeline_cluster_test.go
+++ /dev/null
@@ -1,29 +0,0 @@
-package config_tests
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestOverridePipelineClusterDev(t *testing.T) {
-	b := loadTarget(t, "./override_pipeline_cluster", "development")
-	assert.Equal(t, "job", b.Config.Resources.Pipelines["foo"].Name)
-	assert.Len(t, b.Config.Resources.Pipelines["foo"].Clusters, 1)
-
-	c := b.Config.Resources.Pipelines["foo"].Clusters[0]
-	assert.Equal(t, map[string]string{"foo": "bar"}, c.SparkConf)
-	assert.Equal(t, "i3.xlarge", c.NodeTypeId)
-	assert.Equal(t, 1, c.NumWorkers)
-}
-
-func TestOverridePipelineClusterStaging(t *testing.T) {
-	b := loadTarget(t, "./override_pipeline_cluster", "staging")
-	assert.Equal(t, "job", b.Config.Resources.Pipelines["foo"].Name)
-	assert.Len(t, b.Config.Resources.Pipelines["foo"].Clusters, 1)
-
-	c := b.Config.Resources.Pipelines["foo"].Clusters[0]
-	assert.Equal(t, map[string]string{"foo": "bar"}, c.SparkConf)
-	assert.Equal(t, "i3.2xlarge", c.NodeTypeId)
-	assert.Equal(t, 4, c.NumWorkers)
-}
diff --git a/bundle/tests/variables_test.go b/bundle/tests/variables_test.go
deleted file mode 100644
index 37d488fad..000000000
--- a/bundle/tests/variables_test.go
+++ /dev/null
@@ -1,206 +0,0 @@
-package config_tests
-
-import (
-	"context"
-	"testing"
-
-	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/bundle/config/mutator"
-	"github.com/databricks/databricks-sdk-go/experimental/mocks"
-	"github.com/databricks/databricks-sdk-go/service/compute"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/mock"
-	"github.com/stretchr/testify/require"
-)
-
-func TestVariables(t *testing.T) {
-	t.Setenv("BUNDLE_VAR_b", "def")
-	b := load(t, "./variables/vanilla")
-	diags := bundle.Apply(context.Background(), b, bundle.Seq(
-		mutator.SetVariables(),
-		mutator.ResolveVariableReferences(
-			"variables",
-		),
-	))
-	require.NoError(t, diags.Error())
-	assert.Equal(t, "abc def", b.Config.Bundle.Name)
-}
-
-func TestVariablesLoadingFailsWhenRequiredVariableIsNotSpecified(t *testing.T) {
-	b := load(t, "./variables/vanilla")
-	diags := bundle.Apply(context.Background(), b, bundle.Seq(
-		mutator.SetVariables(),
-		mutator.ResolveVariableReferences(
-			"variables",
-		),
-	))
-	assert.ErrorContains(t, diags.Error(), "no value assigned to required variable b. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_b environment variable")
-}
-
-func TestVariablesTargetsBlockOverride(t *testing.T) {
-	b := load(t, "./variables/env_overrides")
-	diags := bundle.Apply(context.Background(), b, bundle.Seq(
-		mutator.SelectTarget("env-with-single-variable-override"),
-		mutator.SetVariables(),
-		mutator.ResolveVariableReferences(
-			"variables",
-		),
-	))
-	require.NoError(t, diags.Error())
-	assert.Equal(t, "default-a dev-b", b.Config.Workspace.Profile)
-}
-
-func TestVariablesTargetsBlockOverrideForMultipleVariables(t *testing.T) {
-	b := load(t, "./variables/env_overrides")
-	diags := bundle.Apply(context.Background(), b, bundle.Seq(
-		mutator.SelectTarget("env-with-two-variable-overrides"),
-		mutator.SetVariables(),
-		mutator.ResolveVariableReferences(
-			"variables",
-		),
-	))
-	require.NoError(t, diags.Error())
-	assert.Equal(t, "prod-a prod-b", b.Config.Workspace.Profile)
-}
-
-func TestVariablesTargetsBlockOverrideWithProcessEnvVars(t *testing.T) {
-	t.Setenv("BUNDLE_VAR_b", "env-var-b")
-	b := load(t, "./variables/env_overrides")
-	diags := bundle.Apply(context.Background(), b, bundle.Seq(
-		mutator.SelectTarget("env-with-two-variable-overrides"),
-		mutator.SetVariables(),
-		mutator.ResolveVariableReferences(
-			"variables",
-		),
-	))
-	require.NoError(t, diags.Error())
-	assert.Equal(t, "prod-a env-var-b", b.Config.Workspace.Profile)
-}
-
-func TestVariablesTargetsBlockOverrideWithMissingVariables(t *testing.T) {
-	b := load(t, "./variables/env_overrides")
-	diags := bundle.Apply(context.Background(), b, bundle.Seq(
-		mutator.SelectTarget("env-missing-a-required-variable-assignment"),
-		mutator.SetVariables(),
-		mutator.ResolveVariableReferences(
-			"variables",
-		),
-	))
-	assert.ErrorContains(t, diags.Error(), "no value assigned to required variable b. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_b environment variable")
-}
-
-func TestVariablesTargetsBlockOverrideWithUndefinedVariables(t *testing.T) {
-	b := load(t, "./variables/env_overrides")
-	diags := bundle.Apply(context.Background(), b, bundle.Seq(
-		mutator.SelectTarget("env-using-an-undefined-variable"),
-		mutator.SetVariables(),
-		mutator.ResolveVariableReferences(
-			"variables",
-		),
-	))
-	assert.ErrorContains(t, diags.Error(), "variable c is not defined but is assigned a value")
-}
-
-func TestVariablesWithoutDefinition(t *testing.T) {
-	t.Setenv("BUNDLE_VAR_a", "foo")
-	t.Setenv("BUNDLE_VAR_b", "bar")
-	b := load(t, "./variables/without_definition")
-	diags := bundle.Apply(context.Background(), b, mutator.SetVariables())
-	require.NoError(t, diags.Error())
-	require.True(t, b.Config.Variables["a"].HasValue())
-	require.True(t, b.Config.Variables["b"].HasValue())
-	assert.Equal(t, "foo", b.Config.Variables["a"].Value)
-	assert.Equal(t, "bar", b.Config.Variables["b"].Value)
-}
-
-func TestVariablesWithTargetLookupOverrides(t *testing.T) {
-	b := load(t, "./variables/env_overrides")
-
-	mockWorkspaceClient := mocks.NewMockWorkspaceClient(t)
-	b.SetWorkpaceClient(mockWorkspaceClient.WorkspaceClient)
-	instancePoolApi := mockWorkspaceClient.GetMockInstancePoolsAPI()
-	instancePoolApi.EXPECT().GetByInstancePoolName(mock.Anything, "some-test-instance-pool").Return(&compute.InstancePoolAndStats{
-		InstancePoolId: "1234",
-	}, nil)
-
-	clustersApi := mockWorkspaceClient.GetMockClustersAPI()
-	clustersApi.EXPECT().ListAll(mock.Anything, compute.ListClustersRequest{
-		FilterBy: &compute.ListClustersFilterBy{
-			ClusterSources: []compute.ClusterSource{compute.ClusterSourceApi, compute.ClusterSourceUi},
-		},
-	}).Return([]compute.ClusterDetails{
-		{ClusterId: "4321", ClusterName: "some-test-cluster"},
-		{ClusterId: "9876", ClusterName: "some-other-cluster"},
-	}, nil)
-
-	clusterPoliciesApi := mockWorkspaceClient.GetMockClusterPoliciesAPI()
-	clusterPoliciesApi.EXPECT().GetByName(mock.Anything, "some-test-cluster-policy").Return(&compute.Policy{
-		PolicyId: "9876",
-	}, nil)
-
-	diags := bundle.Apply(context.Background(), b, bundle.Seq(
-		mutator.SelectTarget("env-overrides-lookup"),
-		mutator.SetVariables(),
-		mutator.ResolveResourceReferences(),
-	))
-
-	require.NoError(t, diags.Error())
-	assert.Equal(t, "4321", b.Config.Variables["d"].Value)
-	assert.Equal(t, "1234", b.Config.Variables["e"].Value)
-	assert.Equal(t, "9876", b.Config.Variables["f"].Value)
-}
-
-func TestVariableTargetOverrides(t *testing.T) {
-	tcases := []struct {
-		targetName         string
-		pipelineName       string
-		pipelineContinuous bool
-		pipelineNumWorkers int
-	}{
-		{
-			"use-default-variable-values",
-			"a_string",
-			true,
-			42,
-		},
-		{
-			"override-string-variable",
-			"overridden_string",
-			true,
-			42,
-		},
-		{
-			"override-int-variable",
-			"a_string",
-			true,
-			43,
-		},
-		{
-			"override-both-bool-and-string-variables",
-			"overridden_string",
-			false,
-			42,
-		},
-	}
-
-	for _, tcase := range tcases {
-		t.Run(tcase.targetName, func(t *testing.T) {
-			b := loadTarget(t, "./variables/variable_overrides_in_target", tcase.targetName)
-			diags := bundle.Apply(context.Background(), b, bundle.Seq(
-				mutator.SetVariables(),
-				mutator.ResolveVariableReferences("variables")),
-			)
-			require.NoError(t, diags.Error())
-
-			assert.Equal(t, tcase.pipelineName, b.Config.Resources.Pipelines["my_pipeline"].Name)
-			assert.Equal(t, tcase.pipelineContinuous, b.Config.Resources.Pipelines["my_pipeline"].Continuous)
-			assert.Equal(t, tcase.pipelineNumWorkers, b.Config.Resources.Pipelines["my_pipeline"].Clusters[0].NumWorkers)
-		})
-	}
-}
-
-func TestBundleWithEmptyVariableLoads(t *testing.T) {
-	b := load(t, "./variables/empty")
-	diags := bundle.Apply(context.Background(), b, mutator.SetVariables())
-	require.ErrorContains(t, diags.Error(), "no value assigned to required variable a")
-}
diff --git a/libs/env/context.go b/libs/env/context.go
index af4d1afa0..37b76147a 100644
--- a/libs/env/context.go
+++ b/libs/env/context.go
@@ -65,7 +65,7 @@ func Set(ctx context.Context, key, value string) context.Context {
 	return setMap(ctx, m)
 }
 
-func homeEnvVar() string {
+func HomeEnvVar() string {
 	if runtime.GOOS == "windows" {
 		return "USERPROFILE"
 	}
@@ -73,14 +73,14 @@ func homeEnvVar() string {
 }
 
 func WithUserHomeDir(ctx context.Context, value string) context.Context {
-	return Set(ctx, homeEnvVar(), value)
+	return Set(ctx, HomeEnvVar(), value)
 }
 
 // ErrNoHomeEnv indicates the absence of $HOME env variable
 var ErrNoHomeEnv = errors.New("$HOME is not set")
 
 func UserHomeDir(ctx context.Context) (string, error) {
-	home := Get(ctx, homeEnvVar())
+	home := Get(ctx, HomeEnvVar())
 	if home == "" {
 		return "", ErrNoHomeEnv
 	}
diff --git a/libs/testdiff/testdiff.go b/libs/testdiff/testdiff.go
index 71b781362..fef1d5ae2 100644
--- a/libs/testdiff/testdiff.go
+++ b/libs/testdiff/testdiff.go
@@ -25,7 +25,9 @@ func AssertEqualTexts(t testutil.TestingT, filename1, filename2, expected, out s
 	} else {
 		// only show diff for large texts
 		diff := UnifiedDiff(filename1, filename2, expected, out)
-		t.Errorf("Diff:\n" + diff)
+		if diff != "" {
+			t.Errorf("Diff:\n" + diff)
+		}
 	}
 }
 

From 0289becea854dff5a89ee1c14809a3cc3cba998b Mon Sep 17 00:00:00 2001
From: Ilya Kuznetsov <ilya.kuznetsov@databricks.com>
Date: Wed, 8 Jan 2025 13:43:56 +0100
Subject: [PATCH 035/247] Handle `${workspace.file_path}` references in
 source-linked deployments (#2046)

## Changes

1. Updates `workspace.file_path` during source-linked deployment to
address cases like this
https://github.com/databricks/bundle-examples/blob/main/default_python/resources/default_python_pipeline.yml#L13
2. Updates `workspace.file_path` in `metadata.json`
3. Prints warning for users when `workspace.file_path` is explicitly set
but deploy is running in source-linked mode

## Tests

Unit test
---
 bundle/config/mutator/apply_presets.go        |  22 ----
 bundle/config/mutator/apply_presets_test.go   |  88 -------------
 .../apply_source_linked_deployment_preset.go  |  75 +++++++++++
 ...ly_source_linked_deployment_preset_test.go | 122 ++++++++++++++++++
 bundle/config/mutator/process_target_mode.go  |   9 --
 .../mutator/process_target_mode_test.go       |  31 -----
 .../mutator/resolve_variable_references.go    |  25 ++--
 .../resolve_variable_references_test.go       |  55 ++++++++
 bundle/deploy/metadata/compute.go             |   3 +
 bundle/deploy/metadata/compute_test.go        |  21 +++
 bundle/phases/initialize.go                   |   5 +
 bundle/tests/loader.go                        |   3 +-
 libs/template/renderer_test.go                |   3 +-
 13 files changed, 301 insertions(+), 161 deletions(-)
 create mode 100644 bundle/config/mutator/apply_source_linked_deployment_preset.go
 create mode 100644 bundle/config/mutator/apply_source_linked_deployment_preset_test.go

diff --git a/bundle/config/mutator/apply_presets.go b/bundle/config/mutator/apply_presets.go
index 381703756..59b8547be 100644
--- a/bundle/config/mutator/apply_presets.go
+++ b/bundle/config/mutator/apply_presets.go
@@ -9,7 +9,6 @@ import (
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
-	"github.com/databricks/cli/libs/dbr"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/textutil"
@@ -222,27 +221,6 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 		dashboard.DisplayName = prefix + dashboard.DisplayName
 	}
 
-	if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) {
-		isDatabricksWorkspace := dbr.RunsOnRuntime(ctx) && strings.HasPrefix(b.SyncRootPath, "/Workspace/")
-		if !isDatabricksWorkspace {
-			target := b.Config.Bundle.Target
-			path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment"))
-			diags = diags.Append(
-				diag.Diagnostic{
-					Severity: diag.Warning,
-					Summary:  "source-linked deployment is available only in the Databricks Workspace",
-					Paths: []dyn.Path{
-						path,
-					},
-					Locations: b.Config.GetLocations(path[2:].String()),
-				},
-			)
-
-			disabled := false
-			b.Config.Presets.SourceLinkedDeployment = &disabled
-		}
-	}
-
 	return diags
 }
 
diff --git a/bundle/config/mutator/apply_presets_test.go b/bundle/config/mutator/apply_presets_test.go
index c26f20383..5e3f942cc 100644
--- a/bundle/config/mutator/apply_presets_test.go
+++ b/bundle/config/mutator/apply_presets_test.go
@@ -2,16 +2,12 @@ package mutator_test
 
 import (
 	"context"
-	"runtime"
 	"testing"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/mutator"
 	"github.com/databricks/cli/bundle/config/resources"
-	"github.com/databricks/cli/bundle/internal/bundletest"
-	"github.com/databricks/cli/libs/dbr"
-	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/databricks-sdk-go/service/catalog"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/stretchr/testify/require"
@@ -398,87 +394,3 @@ func TestApplyPresetsResourceNotDefined(t *testing.T) {
 		})
 	}
 }
-
-func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
-	}
-
-	testContext := context.Background()
-	enabled := true
-	disabled := false
-	workspacePath := "/Workspace/user.name@company.com"
-
-	tests := []struct {
-		bundlePath      string
-		ctx             context.Context
-		name            string
-		initialValue    *bool
-		expectedValue   *bool
-		expectedWarning string
-	}{
-		{
-			name:          "preset enabled, bundle in Workspace, databricks runtime",
-			bundlePath:    workspacePath,
-			ctx:           dbr.MockRuntime(testContext, true),
-			initialValue:  &enabled,
-			expectedValue: &enabled,
-		},
-		{
-			name:            "preset enabled, bundle not in Workspace, databricks runtime",
-			bundlePath:      "/Users/user.name@company.com",
-			ctx:             dbr.MockRuntime(testContext, true),
-			initialValue:    &enabled,
-			expectedValue:   &disabled,
-			expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
-		},
-		{
-			name:            "preset enabled, bundle in Workspace, not databricks runtime",
-			bundlePath:      workspacePath,
-			ctx:             dbr.MockRuntime(testContext, false),
-			initialValue:    &enabled,
-			expectedValue:   &disabled,
-			expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
-		},
-		{
-			name:          "preset disabled, bundle in Workspace, databricks runtime",
-			bundlePath:    workspacePath,
-			ctx:           dbr.MockRuntime(testContext, true),
-			initialValue:  &disabled,
-			expectedValue: &disabled,
-		},
-		{
-			name:          "preset nil, bundle in Workspace, databricks runtime",
-			bundlePath:    workspacePath,
-			ctx:           dbr.MockRuntime(testContext, true),
-			initialValue:  nil,
-			expectedValue: nil,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			b := &bundle.Bundle{
-				SyncRootPath: tt.bundlePath,
-				Config: config.Root{
-					Presets: config.Presets{
-						SourceLinkedDeployment: tt.initialValue,
-					},
-				},
-			}
-
-			bundletest.SetLocation(b, "presets.source_linked_deployment", []dyn.Location{{File: "databricks.yml"}})
-			diags := bundle.Apply(tt.ctx, b, mutator.ApplyPresets())
-			if diags.HasError() {
-				t.Fatalf("unexpected error: %v", diags)
-			}
-
-			if tt.expectedWarning != "" {
-				require.Equal(t, tt.expectedWarning, diags[0].Summary)
-				require.NotEmpty(t, diags[0].Locations)
-			}
-
-			require.Equal(t, tt.expectedValue, b.Config.Presets.SourceLinkedDeployment)
-		})
-	}
-}
diff --git a/bundle/config/mutator/apply_source_linked_deployment_preset.go b/bundle/config/mutator/apply_source_linked_deployment_preset.go
new file mode 100644
index 000000000..78ccc5322
--- /dev/null
+++ b/bundle/config/mutator/apply_source_linked_deployment_preset.go
@@ -0,0 +1,75 @@
+package mutator
+
+import (
+	"context"
+	"strings"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/libs/dbr"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+)
+
+type applySourceLinkedDeploymentPreset struct{}
+
+// Apply source-linked deployment preset
+func ApplySourceLinkedDeploymentPreset() *applySourceLinkedDeploymentPreset {
+	return &applySourceLinkedDeploymentPreset{}
+}
+
+func (m *applySourceLinkedDeploymentPreset) Name() string {
+	return "ApplySourceLinkedDeploymentPreset"
+}
+
+func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	if config.IsExplicitlyDisabled(b.Config.Presets.SourceLinkedDeployment) {
+		return nil
+	}
+
+	var diags diag.Diagnostics
+	isDatabricksWorkspace := dbr.RunsOnRuntime(ctx) && strings.HasPrefix(b.SyncRootPath, "/Workspace/")
+	target := b.Config.Bundle.Target
+
+	if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) {
+		if !isDatabricksWorkspace {
+			path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment"))
+			diags = diags.Append(
+				diag.Diagnostic{
+					Severity: diag.Warning,
+					Summary:  "source-linked deployment is available only in the Databricks Workspace",
+					Paths: []dyn.Path{
+						path,
+					},
+					Locations: b.Config.GetLocations(path[2:].String()),
+				},
+			)
+
+			disabled := false
+			b.Config.Presets.SourceLinkedDeployment = &disabled
+			return diags
+		}
+	}
+
+	if isDatabricksWorkspace && b.Config.Bundle.Mode == config.Development {
+		enabled := true
+		b.Config.Presets.SourceLinkedDeployment = &enabled
+	}
+
+	if b.Config.Workspace.FilePath != "" && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
+		path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("workspace"), dyn.Key("file_path"))
+
+		diags = diags.Append(
+			diag.Diagnostic{
+				Severity: diag.Warning,
+				Summary:  "workspace.file_path setting will be ignored in source-linked deployment mode",
+				Paths: []dyn.Path{
+					path[2:],
+				},
+				Locations: b.Config.GetLocations(path[2:].String()),
+			},
+		)
+	}
+
+	return diags
+}
diff --git a/bundle/config/mutator/apply_source_linked_deployment_preset_test.go b/bundle/config/mutator/apply_source_linked_deployment_preset_test.go
new file mode 100644
index 000000000..1b74fd8e9
--- /dev/null
+++ b/bundle/config/mutator/apply_source_linked_deployment_preset_test.go
@@ -0,0 +1,122 @@
+package mutator_test
+
+import (
+	"context"
+	"runtime"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/cli/bundle/internal/bundletest"
+	"github.com/databricks/cli/libs/dbr"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/stretchr/testify/require"
+)
+
+func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
+	}
+
+	testContext := context.Background()
+	enabled := true
+	disabled := false
+	workspacePath := "/Workspace/user.name@company.com"
+
+	tests := []struct {
+		name            string
+		ctx             context.Context
+		mutateBundle    func(b *bundle.Bundle)
+		initialValue    *bool
+		expectedValue   *bool
+		expectedWarning string
+	}{
+		{
+			name:          "preset enabled, bundle in Workspace, databricks runtime",
+			ctx:           dbr.MockRuntime(testContext, true),
+			initialValue:  &enabled,
+			expectedValue: &enabled,
+		},
+		{
+			name: "preset enabled, bundle not in Workspace, databricks runtime",
+			ctx:  dbr.MockRuntime(testContext, true),
+			mutateBundle: func(b *bundle.Bundle) {
+				b.SyncRootPath = "/Users/user.name@company.com"
+			},
+			initialValue:    &enabled,
+			expectedValue:   &disabled,
+			expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
+		},
+		{
+			name:            "preset enabled, bundle in Workspace, not databricks runtime",
+			ctx:             dbr.MockRuntime(testContext, false),
+			initialValue:    &enabled,
+			expectedValue:   &disabled,
+			expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
+		},
+		{
+			name:          "preset disabled, bundle in Workspace, databricks runtime",
+			ctx:           dbr.MockRuntime(testContext, true),
+			initialValue:  &disabled,
+			expectedValue: &disabled,
+		},
+		{
+			name:          "preset nil, bundle in Workspace, databricks runtime",
+			ctx:           dbr.MockRuntime(testContext, true),
+			initialValue:  nil,
+			expectedValue: nil,
+		},
+		{
+			name: "preset nil, dev mode true, bundle in Workspace, databricks runtime",
+			ctx:  dbr.MockRuntime(testContext, true),
+			mutateBundle: func(b *bundle.Bundle) {
+				b.Config.Bundle.Mode = config.Development
+			},
+			initialValue:  nil,
+			expectedValue: &enabled,
+		},
+		{
+			name: "preset enabled, workspace.file_path is defined by user",
+			ctx:  dbr.MockRuntime(testContext, true),
+			mutateBundle: func(b *bundle.Bundle) {
+				b.Config.Workspace.FilePath = "file_path"
+			},
+			initialValue:    &enabled,
+			expectedValue:   &enabled,
+			expectedWarning: "workspace.file_path setting will be ignored in source-linked deployment mode",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			b := &bundle.Bundle{
+				SyncRootPath: workspacePath,
+				Config: config.Root{
+					Presets: config.Presets{
+						SourceLinkedDeployment: tt.initialValue,
+					},
+				},
+			}
+
+			if tt.mutateBundle != nil {
+				tt.mutateBundle(b)
+			}
+
+			bundletest.SetLocation(b, "presets.source_linked_deployment", []dyn.Location{{File: "databricks.yml"}})
+			bundletest.SetLocation(b, "workspace.file_path", []dyn.Location{{File: "databricks.yml"}})
+
+			diags := bundle.Apply(tt.ctx, b, mutator.ApplySourceLinkedDeploymentPreset())
+			if diags.HasError() {
+				t.Fatalf("unexpected error: %v", diags)
+			}
+
+			if tt.expectedWarning != "" {
+				require.Equal(t, tt.expectedWarning, diags[0].Summary)
+				require.NotEmpty(t, diags[0].Locations)
+			}
+
+			require.Equal(t, tt.expectedValue, b.Config.Presets.SourceLinkedDeployment)
+		})
+	}
+}
diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go
index df0136fad..44b53681d 100644
--- a/bundle/config/mutator/process_target_mode.go
+++ b/bundle/config/mutator/process_target_mode.go
@@ -6,7 +6,6 @@ import (
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
-	"github.com/databricks/cli/libs/dbr"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/iamutil"
@@ -58,14 +57,6 @@ func transformDevelopmentMode(ctx context.Context, b *bundle.Bundle) {
 		t.TriggerPauseStatus = config.Paused
 	}
 
-	if !config.IsExplicitlyDisabled(t.SourceLinkedDeployment) {
-		isInWorkspace := strings.HasPrefix(b.SyncRootPath, "/Workspace/")
-		if isInWorkspace && dbr.RunsOnRuntime(ctx) {
-			enabled := true
-			t.SourceLinkedDeployment = &enabled
-		}
-	}
-
 	if !config.IsExplicitlyDisabled(t.PipelinesDevelopment) {
 		enabled := true
 		t.PipelinesDevelopment = &enabled
diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go
index c299a7636..097c092a6 100644
--- a/bundle/config/mutator/process_target_mode_test.go
+++ b/bundle/config/mutator/process_target_mode_test.go
@@ -3,14 +3,12 @@ package mutator
 import (
 	"context"
 	"reflect"
-	"runtime"
 	"slices"
 	"testing"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/resources"
-	"github.com/databricks/cli/libs/dbr"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/tags"
 	"github.com/databricks/cli/libs/vfs"
@@ -540,32 +538,3 @@ func TestPipelinesDevelopmentDisabled(t *testing.T) {
 
 	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
 }
-
-func TestSourceLinkedDeploymentEnabled(t *testing.T) {
-	b, diags := processSourceLinkedBundle(t, true)
-	require.NoError(t, diags.Error())
-	assert.True(t, *b.Config.Presets.SourceLinkedDeployment)
-}
-
-func TestSourceLinkedDeploymentDisabled(t *testing.T) {
-	b, diags := processSourceLinkedBundle(t, false)
-	require.NoError(t, diags.Error())
-	assert.False(t, *b.Config.Presets.SourceLinkedDeployment)
-}
-
-func processSourceLinkedBundle(t *testing.T, presetEnabled bool) (*bundle.Bundle, diag.Diagnostics) {
-	if runtime.GOOS == "windows" {
-		t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
-	}
-
-	b := mockBundle(config.Development)
-
-	workspacePath := "/Workspace/lennart@company.com/"
-	b.SyncRootPath = workspacePath
-	b.Config.Presets.SourceLinkedDeployment = &presetEnabled
-
-	ctx := dbr.MockRuntime(context.Background(), true)
-	m := bundle.Seq(ProcessTargetMode(), ApplyPresets())
-	diags := bundle.Apply(ctx, b, m)
-	return b, diags
-}
diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go
index e074c2b84..7ad3dfd8d 100644
--- a/bundle/config/mutator/resolve_variable_references.go
+++ b/bundle/config/mutator/resolve_variable_references.go
@@ -5,6 +5,7 @@ import (
 	"errors"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/variable"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
@@ -15,7 +16,7 @@ import (
 type resolveVariableReferences struct {
 	prefixes []string
 	pattern  dyn.Pattern
-	lookupFn func(dyn.Value, dyn.Path) (dyn.Value, error)
+	lookupFn func(dyn.Value, dyn.Path, *bundle.Bundle) (dyn.Value, error)
 	skipFn   func(dyn.Value) bool
 }
 
@@ -44,16 +45,21 @@ func ResolveVariableReferencesInComplexVariables() bundle.Mutator {
 	}
 }
 
-func lookup(v dyn.Value, path dyn.Path) (dyn.Value, error) {
+func lookup(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) {
+	if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
+		if path.String() == "workspace.file_path" {
+			return dyn.V(b.SyncRootPath), nil
+		}
+	}
 	// Future opportunity: if we lookup this path in both the given root
 	// and the synthesized root, we know if it was explicitly set or implied to be empty.
 	// Then we can emit a warning if it was not explicitly set.
 	return dyn.GetByPath(v, path)
 }
 
-func lookupForComplexVariables(v dyn.Value, path dyn.Path) (dyn.Value, error) {
+func lookupForComplexVariables(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) {
 	if path[0].Key() != "variables" {
-		return lookup(v, path)
+		return lookup(v, path, b)
 	}
 
 	varV, err := dyn.GetByPath(v, path[:len(path)-1])
@@ -71,7 +77,7 @@ func lookupForComplexVariables(v dyn.Value, path dyn.Path) (dyn.Value, error) {
 		return dyn.InvalidValue, errors.New("complex variables cannot contain references to another complex variables")
 	}
 
-	return lookup(v, path)
+	return lookup(v, path, b)
 }
 
 func skipResolvingInNonComplexVariables(v dyn.Value) bool {
@@ -83,9 +89,9 @@ func skipResolvingInNonComplexVariables(v dyn.Value) bool {
 	}
 }
 
-func lookupForVariables(v dyn.Value, path dyn.Path) (dyn.Value, error) {
+func lookupForVariables(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) {
 	if path[0].Key() != "variables" {
-		return lookup(v, path)
+		return lookup(v, path, b)
 	}
 
 	varV, err := dyn.GetByPath(v, path[:len(path)-1])
@@ -103,7 +109,7 @@ func lookupForVariables(v dyn.Value, path dyn.Path) (dyn.Value, error) {
 		return dyn.InvalidValue, errors.New("lookup variables cannot contain references to another lookup variables")
 	}
 
-	return lookup(v, path)
+	return lookup(v, path, b)
 }
 
 func (*resolveVariableReferences) Name() string {
@@ -125,6 +131,7 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle)
 	varPath := dyn.NewPath(dyn.Key("var"))
 
 	var diags diag.Diagnostics
+
 	err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
 		// Synthesize a copy of the root that has all fields that are present in the type
 		// but not set in the dynamic value set to their corresponding empty value.
@@ -167,7 +174,7 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle)
 						if m.skipFn != nil && m.skipFn(v) {
 							return dyn.InvalidValue, dynvar.ErrSkipResolution
 						}
-						return m.lookupFn(normalized, path)
+						return m.lookupFn(normalized, path, b)
 					}
 				}
 
diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go
index 07972ecf4..fcad3180e 100644
--- a/bundle/config/mutator/resolve_variable_references_test.go
+++ b/bundle/config/mutator/resolve_variable_references_test.go
@@ -12,6 +12,7 @@ import (
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/databricks/databricks-sdk-go/service/pipelines"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -434,3 +435,57 @@ func TestResolveComplexVariableWithVarReference(t *testing.T) {
 	require.NoError(t, diags.Error())
 	require.Equal(t, "cicd_template==1.0.0", b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].Libraries[0].Pypi.Package)
 }
+
+func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) {
+	testCases := []struct {
+		enabled bool
+		assert  func(t *testing.T, b *bundle.Bundle)
+	}{
+		{
+			true,
+			func(t *testing.T, b *bundle.Bundle) {
+				// Variables that use workspace file path should have SyncRootValue during resolution phase
+				require.Equal(t, "sync/root/path", b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Configuration["source"])
+
+				// The file path itself should remain the same
+				require.Equal(t, "file/path", b.Config.Workspace.FilePath)
+			},
+		},
+		{
+			false,
+			func(t *testing.T, b *bundle.Bundle) {
+				require.Equal(t, "file/path", b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Configuration["source"])
+				require.Equal(t, "file/path", b.Config.Workspace.FilePath)
+			},
+		},
+	}
+
+	for _, testCase := range testCases {
+		b := &bundle.Bundle{
+			SyncRootPath: "sync/root/path",
+			Config: config.Root{
+				Presets: config.Presets{
+					SourceLinkedDeployment: &testCase.enabled,
+				},
+				Workspace: config.Workspace{
+					FilePath: "file/path",
+				},
+				Resources: config.Resources{
+					Pipelines: map[string]*resources.Pipeline{
+						"pipeline1": {
+							PipelineSpec: &pipelines.PipelineSpec{
+								Configuration: map[string]string{
+									"source": "${workspace.file_path}",
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+
+		diags := bundle.Apply(context.Background(), b, ResolveVariableReferences("workspace"))
+		require.NoError(t, diags.Error())
+		testCase.assert(t, b)
+	}
+}
diff --git a/bundle/deploy/metadata/compute.go b/bundle/deploy/metadata/compute.go
index bc8767de4..b47baa6b2 100644
--- a/bundle/deploy/metadata/compute.go
+++ b/bundle/deploy/metadata/compute.go
@@ -54,5 +54,8 @@ func (m *compute) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
 
 	// Set file upload destination of the bundle in metadata
 	b.Metadata.Config.Workspace.FilePath = b.Config.Workspace.FilePath
+	if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
+		b.Metadata.Config.Workspace.FilePath = b.SyncRootPath
+	}
 	return nil
 }
diff --git a/bundle/deploy/metadata/compute_test.go b/bundle/deploy/metadata/compute_test.go
index 2c2c72376..c6fa9bddb 100644
--- a/bundle/deploy/metadata/compute_test.go
+++ b/bundle/deploy/metadata/compute_test.go
@@ -97,3 +97,24 @@ func TestComputeMetadataMutator(t *testing.T) {
 
 	assert.Equal(t, expectedMetadata, b.Metadata)
 }
+
+func TestComputeMetadataMutatorSourceLinked(t *testing.T) {
+	syncRootPath := "/Users/shreyas.goenka@databricks.com/source"
+	enabled := true
+	b := &bundle.Bundle{
+		SyncRootPath: syncRootPath,
+		Config: config.Root{
+			Presets: config.Presets{
+				SourceLinkedDeployment: &enabled,
+			},
+			Workspace: config.Workspace{
+				FilePath: "/Users/shreyas.goenka@databricks.com/files",
+			},
+		},
+	}
+
+	diags := bundle.Apply(context.Background(), b, Compute())
+	require.NoError(t, diags.Error())
+
+	assert.Equal(t, syncRootPath, b.Metadata.Config.Workspace.FilePath)
+}
diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go
index f0cbc00c2..f7b3cd608 100644
--- a/bundle/phases/initialize.go
+++ b/bundle/phases/initialize.go
@@ -41,6 +41,10 @@ func Initialize() bundle.Mutator {
 			mutator.PopulateCurrentUser(),
 			mutator.LoadGitDetails(),
 
+			// This mutator needs to be run before variable interpolation and defining default workspace paths
+			// because it affects how workspace variables are resolved.
+			mutator.ApplySourceLinkedDeploymentPreset(),
+
 			mutator.DefineDefaultWorkspaceRoot(),
 			mutator.ExpandWorkspaceRoot(),
 			mutator.DefineDefaultWorkspacePaths(),
@@ -51,6 +55,7 @@ func Initialize() bundle.Mutator {
 			mutator.RewriteWorkspacePrefix(),
 
 			mutator.SetVariables(),
+
 			// Intentionally placed before ResolveVariableReferencesInLookup, ResolveResourceReferences,
 			// ResolveVariableReferencesInComplexVariables and ResolveVariableReferences.
 			// See what is expected in PythonMutatorPhaseInit doc
diff --git a/bundle/tests/loader.go b/bundle/tests/loader.go
index 5c48d81cb..bb68b3059 100644
--- a/bundle/tests/loader.go
+++ b/bundle/tests/loader.go
@@ -7,6 +7,7 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config/mutator"
 	"github.com/databricks/cli/bundle/phases"
+	"github.com/databricks/cli/libs/dbr"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/databricks-sdk-go/config"
 	"github.com/databricks/databricks-sdk-go/experimental/mocks"
@@ -66,7 +67,7 @@ func initializeTarget(t *testing.T, path, env string) (*bundle.Bundle, diag.Diag
 	b := load(t, path)
 	configureMock(t, b)
 
-	ctx := context.Background()
+	ctx := dbr.MockRuntime(context.Background(), false)
 	diags := bundle.Apply(ctx, b, bundle.Seq(
 		mutator.SelectTarget(env),
 		phases.Initialize(),
diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go
index 70c8de12b..7ec1c5ced 100644
--- a/libs/template/renderer_test.go
+++ b/libs/template/renderer_test.go
@@ -16,6 +16,7 @@ import (
 	"github.com/databricks/cli/bundle/phases"
 	"github.com/databricks/cli/cmd/root"
 	"github.com/databricks/cli/internal/testutil"
+	"github.com/databricks/cli/libs/dbr"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/filer"
 	"github.com/databricks/cli/libs/tags"
@@ -39,7 +40,7 @@ func assertFilePermissions(t *testing.T, path string, perm fs.FileMode) {
 }
 
 func assertBuiltinTemplateValid(t *testing.T, template string, settings map[string]any, target string, isServicePrincipal, build bool, tempDir string) {
-	ctx := context.Background()
+	ctx := dbr.MockRuntime(context.Background(), false)
 
 	templateFS, err := fs.Sub(builtinTemplates, path.Join("templates", template))
 	require.NoError(t, err)

From 23f05f5d67cbfc0b41a4ab60de101aaf4bd69ca4 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Wed, 8 Jan 2025 14:18:28 +0100
Subject: [PATCH 036/247] Set the write bit for files written during template
 initialization (#2068)

## Changes

This used to work because the permission bits for built-in templates
were hardcoded to 0644 for files and 0755 for directories.

As of #1912 (and the PRs it depends on), built-in templates are no
longer pre-materialized to a temporary directory and read directly from
the embedded filesystem. This built-in filesystem returns 0444 as the
permission bits for the files it contains. These bits are carried over
to the destination filesystem.

This change updates template materialization to always set the owner's
write bit. It doesn't really make sense to write read-only files and
expect users to work with these files in a VCS (note: Git only stores
the executable bit).

The regression shipped as part of v0.235.0 and will be fixed as of
v0.238.0.

## Tests

Unit tests.
---
 internal/testutil/file.go      | 29 ++++++++++++++++++++++++++
 libs/template/file_test.go     |  9 +++++----
 libs/template/renderer.go      |  4 ++++
 libs/template/renderer_test.go | 37 ++++++++++++++++++++--------------
 4 files changed, 60 insertions(+), 19 deletions(-)

diff --git a/internal/testutil/file.go b/internal/testutil/file.go
index 538a3c20a..476c4123a 100644
--- a/internal/testutil/file.go
+++ b/internal/testutil/file.go
@@ -4,6 +4,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -52,3 +53,31 @@ func ReadFile(t TestingT, path string) string {
 
 	return string(b)
 }
+
+// StatFile returns the file info for a file.
+func StatFile(t TestingT, path string) os.FileInfo {
+	fi, err := os.Stat(path)
+	require.NoError(t, err)
+
+	return fi
+}
+
+// AssertFileContents asserts that the file at path has the expected content.
+func AssertFileContents(t TestingT, path, expected string) bool {
+	actual := ReadFile(t, path)
+	return assert.Equal(t, expected, actual)
+}
+
+// AssertFilePermissions asserts that the file at path has the expected permissions.
+func AssertFilePermissions(t TestingT, path string, expected os.FileMode) bool {
+	fi := StatFile(t, path)
+	assert.False(t, fi.Mode().IsDir(), "expected a file, got a directory")
+	return assert.Equal(t, expected, fi.Mode().Perm(), "expected 0%o, got 0%o", expected, fi.Mode().Perm())
+}
+
+// AssertDirPermissions asserts that the file at path has the expected permissions.
+func AssertDirPermissions(t TestingT, path string, expected os.FileMode) bool {
+	fi := StatFile(t, path)
+	assert.True(t, fi.Mode().IsDir(), "expected a directory, got a file")
+	return assert.Equal(t, expected, fi.Mode().Perm(), "expected 0%o, got 0%o", expected, fi.Mode().Perm())
+}
diff --git a/libs/template/file_test.go b/libs/template/file_test.go
index ced38c284..f4bf5652c 100644
--- a/libs/template/file_test.go
+++ b/libs/template/file_test.go
@@ -8,6 +8,7 @@ import (
 	"runtime"
 	"testing"
 
+	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/filer"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -27,8 +28,8 @@ func testInMemoryFile(t *testing.T, ctx context.Context, perm fs.FileMode) {
 	err = f.Write(ctx, out)
 	assert.NoError(t, err)
 
-	assertFileContent(t, filepath.Join(tmpDir, "a/b/c"), "123")
-	assertFilePermissions(t, filepath.Join(tmpDir, "a/b/c"), perm)
+	testutil.AssertFileContents(t, filepath.Join(tmpDir, "a/b/c"), "123")
+	testutil.AssertFilePermissions(t, filepath.Join(tmpDir, "a/b/c"), perm)
 }
 
 func testCopyFile(t *testing.T, ctx context.Context, perm fs.FileMode) {
@@ -48,8 +49,8 @@ func testCopyFile(t *testing.T, ctx context.Context, perm fs.FileMode) {
 	err = f.Write(ctx, out)
 	assert.NoError(t, err)
 
-	assertFileContent(t, filepath.Join(tmpDir, "a/b/c"), "qwerty")
-	assertFilePermissions(t, filepath.Join(tmpDir, "a/b/c"), perm)
+	testutil.AssertFileContents(t, filepath.Join(tmpDir, "source"), "qwerty")
+	testutil.AssertFilePermissions(t, filepath.Join(tmpDir, "source"), perm)
 }
 
 func TestTemplateInMemoryFilePersistToDisk(t *testing.T) {
diff --git a/libs/template/renderer.go b/libs/template/renderer.go
index 5030cd9df..679b7d8b7 100644
--- a/libs/template/renderer.go
+++ b/libs/template/renderer.go
@@ -150,6 +150,10 @@ func (r *renderer) computeFile(relPathTemplate string) (file, error) {
 	}
 	perm := info.Mode().Perm()
 
+	// Always include the write bit for the owner of the file.
+	// It does not make sense to have a file that is not writable by the owner.
+	perm |= 0o200
+
 	// Execute relative path template to get destination path for the file
 	relPath, err := r.executeTemplate(relPathTemplate)
 	if err != nil {
diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go
index 7ec1c5ced..b2ec388bd 100644
--- a/libs/template/renderer_test.go
+++ b/libs/template/renderer_test.go
@@ -27,16 +27,19 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func assertFileContent(t *testing.T, path, content string) {
-	b, err := os.ReadFile(path)
-	require.NoError(t, err)
-	assert.Equal(t, content, string(b))
-}
+var (
+	defaultFilePermissions fs.FileMode
+	defaultDirPermissions  fs.FileMode
+)
 
-func assertFilePermissions(t *testing.T, path string, perm fs.FileMode) {
-	info, err := os.Stat(path)
-	require.NoError(t, err)
-	assert.Equal(t, perm, info.Mode().Perm())
+func init() {
+	if runtime.GOOS == "windows" {
+		defaultFilePermissions = fs.FileMode(0o666)
+		defaultDirPermissions = fs.FileMode(0o777)
+	} else {
+		defaultFilePermissions = fs.FileMode(0o644)
+		defaultDirPermissions = fs.FileMode(0o755)
+	}
 }
 
 func assertBuiltinTemplateValid(t *testing.T, template string, settings map[string]any, target string, isServicePrincipal, build bool, tempDir string) {
@@ -69,6 +72,10 @@ func assertBuiltinTemplateValid(t *testing.T, template string, settings map[stri
 	err = renderer.persistToDisk(ctx, out)
 	require.NoError(t, err)
 
+	// Verify permissions on file and directory
+	testutil.AssertFilePermissions(t, filepath.Join(tempDir, "my_project/README.md"), defaultFilePermissions)
+	testutil.AssertDirPermissions(t, filepath.Join(tempDir, "my_project/resources"), defaultDirPermissions)
+
 	b, err := bundle.Load(ctx, filepath.Join(tempDir, "my_project"))
 	require.NoError(t, err)
 	diags := bundle.Apply(ctx, b, phases.LoadNamedTarget(target))
@@ -347,10 +354,10 @@ func TestRendererPersistToDisk(t *testing.T) {
 	assert.NoFileExists(t, filepath.Join(tmpDir, "a", "b", "c"))
 	assert.NoFileExists(t, filepath.Join(tmpDir, "mno"))
 
-	assertFileContent(t, filepath.Join(tmpDir, "a", "b", "d"), "123")
-	assertFilePermissions(t, filepath.Join(tmpDir, "a", "b", "d"), 0o444)
-	assertFileContent(t, filepath.Join(tmpDir, "mmnn"), "456")
-	assertFilePermissions(t, filepath.Join(tmpDir, "mmnn"), 0o444)
+	testutil.AssertFileContents(t, filepath.Join(tmpDir, "a/b/d"), "123")
+	testutil.AssertFilePermissions(t, filepath.Join(tmpDir, "a/b/d"), fs.FileMode(0o444))
+	testutil.AssertFileContents(t, filepath.Join(tmpDir, "mmnn"), "456")
+	testutil.AssertFilePermissions(t, filepath.Join(tmpDir, "mmnn"), fs.FileMode(0o444))
 }
 
 func TestRendererWalk(t *testing.T) {
@@ -617,8 +624,8 @@ func TestRendererFileTreeRendering(t *testing.T) {
 	require.NoError(t, err)
 
 	// Assert files and directories are correctly materialized.
-	assert.DirExists(t, filepath.Join(tmpDir, "my_directory"))
-	assert.FileExists(t, filepath.Join(tmpDir, "my_directory", "my_file"))
+	testutil.AssertDirPermissions(t, filepath.Join(tmpDir, "my_directory"), defaultDirPermissions)
+	testutil.AssertFilePermissions(t, filepath.Join(tmpDir, "my_directory", "my_file"), defaultFilePermissions)
 }
 
 func TestRendererSubTemplateInPath(t *testing.T) {

From b48fa70984dc07273110c7309c239aa147d8bfbe Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Wed, 8 Jan 2025 15:47:38 +0100
Subject: [PATCH 037/247] [Release] Release v0.238.0 (#2096)

Bundles:
* Fix finding Python within virtualenv on Windows
([#2034](https://github.com/databricks/cli/pull/2034)).
* Include missing field descriptions in JSON schema
([#2045](https://github.com/databricks/cli/pull/2045)).
* Add validation for volume referenced from `artifact_path`
([#2050](https://github.com/databricks/cli/pull/2050)).
* Handle `${workspace.file_path}` references in source-linked
deployments ([#2046](https://github.com/databricks/cli/pull/2046)).
* Set the write bit for files written during template initialization
([#2068](https://github.com/databricks/cli/pull/2068)).
---
 CHANGELOG.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6bdb0795b..5b59fa540 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,14 @@
 # Version changelog
 
+## [Release] Release v0.238.0
+
+Bundles:
+ * Fix finding Python within virtualenv on Windows ([#2034](https://github.com/databricks/cli/pull/2034)).
+ * Include missing field descriptions in JSON schema ([#2045](https://github.com/databricks/cli/pull/2045)).
+ * Add validation for volume referenced from `artifact_path` ([#2050](https://github.com/databricks/cli/pull/2050)).
+ * Handle `${workspace.file_path}` references in source-linked deployments ([#2046](https://github.com/databricks/cli/pull/2046)).
+ * Set the write bit for files written during template initialization ([#2068](https://github.com/databricks/cli/pull/2068)).
+
 ## [Release] Release v0.237.0
 
 Bundles:

From 42b34c7befcbd2f2cfc742bb61c72a0b0cdca5f8 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Wed, 8 Jan 2025 16:50:44 +0100
Subject: [PATCH 038/247] Update runner for the create-xyz-pr jobs in the
 release workflow (#2098)

## Changes

I missed these in #2077 and they failed because of it on the v0.238.0
release.
---
 .github/workflows/release.yml | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 88e338a8c..061688506 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -58,8 +58,12 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   create-setup-cli-release-pr:
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     needs: goreleaser
-    runs-on: ubuntu-latest
+
     steps:
       - name: Set VERSION variable from tag
         run: |
@@ -82,8 +86,12 @@ jobs:
             });
 
   create-homebrew-tap-release-pr:
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     needs: goreleaser
-    runs-on: ubuntu-latest
+
     steps:
       - name: Set VERSION variable from tag
         run: |
@@ -119,8 +127,12 @@ jobs:
             });
 
   create-vscode-extension-update-pr:
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     needs: goreleaser
-    runs-on: ubuntu-latest
+
     steps:
       - name: Set VERSION variable from tag
         run: |

From df17e4b4ea1da0d46a61f07eef9a2c6d886a1c0d Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 8 Jan 2025 18:44:52 +0100
Subject: [PATCH 039/247] Convert some resolve variables tests to acceptance
 test (#2100)

---
 .../variables/resolve-builtin/databricks.yml  |   6 +
 .../variables/resolve-builtin/output.txt      |  11 ++
 .../bundle/variables/resolve-builtin/script   |   1 +
 .../variables/resolve-empty/databricks.yml    |  10 ++
 .../bundle/variables/resolve-empty/output.txt |   3 +
 .../bundle/variables/resolve-empty/script     |   1 +
 .../databricks.yml                            |  16 +++
 .../resolve-field-within-complex/output.txt   |   3 +
 .../resolve-field-within-complex/script       |   1 +
 .../resolve_variable_references_test.go       | 114 ------------------
 10 files changed, 52 insertions(+), 114 deletions(-)
 create mode 100644 acceptance/bundle/variables/resolve-builtin/databricks.yml
 create mode 100644 acceptance/bundle/variables/resolve-builtin/output.txt
 create mode 100644 acceptance/bundle/variables/resolve-builtin/script
 create mode 100644 acceptance/bundle/variables/resolve-empty/databricks.yml
 create mode 100644 acceptance/bundle/variables/resolve-empty/output.txt
 create mode 100644 acceptance/bundle/variables/resolve-empty/script
 create mode 100644 acceptance/bundle/variables/resolve-field-within-complex/databricks.yml
 create mode 100644 acceptance/bundle/variables/resolve-field-within-complex/output.txt
 create mode 100644 acceptance/bundle/variables/resolve-field-within-complex/script

diff --git a/acceptance/bundle/variables/resolve-builtin/databricks.yml b/acceptance/bundle/variables/resolve-builtin/databricks.yml
new file mode 100644
index 000000000..4bb71c8db
--- /dev/null
+++ b/acceptance/bundle/variables/resolve-builtin/databricks.yml
@@ -0,0 +1,6 @@
+bundle:
+  name: TestResolveVariableReferences
+
+workspace:
+  root_path: "${bundle.name}/bar"
+  file_path: "${workspace.root_path}/baz"
diff --git a/acceptance/bundle/variables/resolve-builtin/output.txt b/acceptance/bundle/variables/resolve-builtin/output.txt
new file mode 100644
index 000000000..2f58abd8a
--- /dev/null
+++ b/acceptance/bundle/variables/resolve-builtin/output.txt
@@ -0,0 +1,11 @@
+{
+  "artifact_path": "TestResolveVariableReferences/bar/artifacts",
+  "current_user": {
+    "short_name": "tester",
+    "userName": "tester@databricks.com"
+  },
+  "file_path": "TestResolveVariableReferences/bar/baz",
+  "resource_path": "TestResolveVariableReferences/bar/resources",
+  "root_path": "TestResolveVariableReferences/bar",
+  "state_path": "TestResolveVariableReferences/bar/state"
+}
diff --git a/acceptance/bundle/variables/resolve-builtin/script b/acceptance/bundle/variables/resolve-builtin/script
new file mode 100644
index 000000000..fefd9abe6
--- /dev/null
+++ b/acceptance/bundle/variables/resolve-builtin/script
@@ -0,0 +1 @@
+$CLI bundle validate -o json | jq .workspace
diff --git a/acceptance/bundle/variables/resolve-empty/databricks.yml b/acceptance/bundle/variables/resolve-empty/databricks.yml
new file mode 100644
index 000000000..7563ada34
--- /dev/null
+++ b/acceptance/bundle/variables/resolve-empty/databricks.yml
@@ -0,0 +1,10 @@
+bundle:
+  name: TestResolveVariableReferencesToEmptyFields
+  git:
+    branch: ""
+
+resources:
+  jobs:
+    job1:
+      tags:
+        git_branch: "${bundle.git.branch}"
diff --git a/acceptance/bundle/variables/resolve-empty/output.txt b/acceptance/bundle/variables/resolve-empty/output.txt
new file mode 100644
index 000000000..a05cbbf54
--- /dev/null
+++ b/acceptance/bundle/variables/resolve-empty/output.txt
@@ -0,0 +1,3 @@
+{
+  "git_branch": ""
+}
diff --git a/acceptance/bundle/variables/resolve-empty/script b/acceptance/bundle/variables/resolve-empty/script
new file mode 100644
index 000000000..614673054
--- /dev/null
+++ b/acceptance/bundle/variables/resolve-empty/script
@@ -0,0 +1 @@
+$CLI bundle validate -o json | jq .resources.jobs.job1.tags
diff --git a/acceptance/bundle/variables/resolve-field-within-complex/databricks.yml b/acceptance/bundle/variables/resolve-field-within-complex/databricks.yml
new file mode 100644
index 000000000..7250dd5df
--- /dev/null
+++ b/acceptance/bundle/variables/resolve-field-within-complex/databricks.yml
@@ -0,0 +1,16 @@
+bundle:
+  name: TestResolveComplexVariableReferencesToFields
+
+variables:
+  cluster:
+    type: "complex"
+    default:
+      node_type_id: "Standard_DS3_v2"
+      num_workers: 2
+
+resources:
+  jobs:
+    job1:
+      job_clusters:
+        - new_cluster:
+            node_type_id: "${var.cluster.node_type_id}"
diff --git a/acceptance/bundle/variables/resolve-field-within-complex/output.txt b/acceptance/bundle/variables/resolve-field-within-complex/output.txt
new file mode 100644
index 000000000..1f6bdbbf4
--- /dev/null
+++ b/acceptance/bundle/variables/resolve-field-within-complex/output.txt
@@ -0,0 +1,3 @@
+{
+  "node_type_id": "Standard_DS3_v2"
+}
diff --git a/acceptance/bundle/variables/resolve-field-within-complex/script b/acceptance/bundle/variables/resolve-field-within-complex/script
new file mode 100644
index 000000000..a885870a5
--- /dev/null
+++ b/acceptance/bundle/variables/resolve-field-within-complex/script
@@ -0,0 +1 @@
+$CLI bundle validate -o json | jq .resources.jobs.job1.job_clusters[0].new_cluster
diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go
index fcad3180e..18bb022aa 100644
--- a/bundle/config/mutator/resolve_variable_references_test.go
+++ b/bundle/config/mutator/resolve_variable_references_test.go
@@ -17,32 +17,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestResolveVariableReferences(t *testing.T) {
-	b := &bundle.Bundle{
-		Config: config.Root{
-			Bundle: config.Bundle{
-				Name: "example",
-			},
-			Workspace: config.Workspace{
-				RootPath: "${bundle.name}/bar",
-				FilePath: "${workspace.root_path}/baz",
-			},
-		},
-	}
-
-	// Apply with an invalid prefix. This should not change the workspace root path.
-	diags := bundle.Apply(context.Background(), b, ResolveVariableReferences("doesntexist"))
-	require.NoError(t, diags.Error())
-	require.Equal(t, "${bundle.name}/bar", b.Config.Workspace.RootPath)
-	require.Equal(t, "${workspace.root_path}/baz", b.Config.Workspace.FilePath)
-
-	// Apply with a valid prefix. This should change the workspace root path.
-	diags = bundle.Apply(context.Background(), b, ResolveVariableReferences("bundle", "workspace"))
-	require.NoError(t, diags.Error())
-	require.Equal(t, "example/bar", b.Config.Workspace.RootPath)
-	require.Equal(t, "example/bar/baz", b.Config.Workspace.FilePath)
-}
-
 func TestResolveVariableReferencesToBundleVariables(t *testing.T) {
 	b := &bundle.Bundle{
 		Config: config.Root{
@@ -66,37 +40,6 @@ func TestResolveVariableReferencesToBundleVariables(t *testing.T) {
 	require.Equal(t, "example/bar", b.Config.Workspace.RootPath)
 }
 
-func TestResolveVariableReferencesToEmptyFields(t *testing.T) {
-	b := &bundle.Bundle{
-		Config: config.Root{
-			Bundle: config.Bundle{
-				Name: "example",
-				Git: config.Git{
-					Branch: "",
-				},
-			},
-			Resources: config.Resources{
-				Jobs: map[string]*resources.Job{
-					"job1": {
-						JobSettings: &jobs.JobSettings{
-							Tags: map[string]string{
-								"git_branch": "${bundle.git.branch}",
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-
-	// Apply for the bundle prefix.
-	diags := bundle.Apply(context.Background(), b, ResolveVariableReferences("bundle"))
-	require.NoError(t, diags.Error())
-
-	// The job settings should have been interpolated to an empty string.
-	require.Equal(t, "", b.Config.Resources.Jobs["job1"].JobSettings.Tags["git_branch"])
-}
-
 func TestResolveVariableReferencesForPrimitiveNonStringFields(t *testing.T) {
 	var diags diag.Diagnostics
 
@@ -251,63 +194,6 @@ func TestResolveComplexVariable(t *testing.T) {
 	require.Equal(t, 2, b.Config.Resources.Jobs["job1"].JobSettings.JobClusters[0].NewCluster.NumWorkers)
 }
 
-func TestResolveComplexVariableReferencesToFields(t *testing.T) {
-	b := &bundle.Bundle{
-		Config: config.Root{
-			Bundle: config.Bundle{
-				Name: "example",
-			},
-			Variables: map[string]*variable.Variable{
-				"cluster": {
-					Value: map[string]any{
-						"node_type_id": "Standard_DS3_v2",
-						"num_workers":  2,
-					},
-					Type: variable.VariableTypeComplex,
-				},
-			},
-
-			Resources: config.Resources{
-				Jobs: map[string]*resources.Job{
-					"job1": {
-						JobSettings: &jobs.JobSettings{
-							JobClusters: []jobs.JobCluster{
-								{
-									NewCluster: compute.ClusterSpec{
-										NodeTypeId: "random",
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-
-	ctx := context.Background()
-
-	// Assign the variables to the dynamic configuration.
-	diags := bundle.ApplyFunc(ctx, b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-		err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
-			var p dyn.Path
-			var err error
-
-			p = dyn.MustPathFromString("resources.jobs.job1.job_clusters[0].new_cluster")
-			v, err = dyn.SetByPath(v, p.Append(dyn.Key("node_type_id")), dyn.V("${var.cluster.node_type_id}"))
-			require.NoError(t, err)
-
-			return v, nil
-		})
-		return diag.FromErr(err)
-	})
-	require.NoError(t, diags.Error())
-
-	diags = bundle.Apply(ctx, b, ResolveVariableReferences("bundle", "workspace", "variables"))
-	require.NoError(t, diags.Error())
-	require.Equal(t, "Standard_DS3_v2", b.Config.Resources.Jobs["job1"].JobSettings.JobClusters[0].NewCluster.NodeTypeId)
-}
-
 func TestResolveComplexVariableReferencesWithComplexVariablesError(t *testing.T) {
 	b := &bundle.Bundle{
 		Config: config.Root{

From b0706ccdc129291714d46de408fbed4c7baca733 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 9 Jan 2025 10:00:05 +0100
Subject: [PATCH 040/247] Use -update instead of TESTS_OUTPUT=OVERWRITE (#2097)

It's easier to remember and type and also validated and part of help:

```
  % go test ./acceptance -updat 2>&1 | grep updat
  flag provided but not defined: -updat
    -update
```
---
 acceptance/README.md    | 2 +-
 libs/testdiff/golden.go | 7 ++++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/acceptance/README.md b/acceptance/README.md
index 162c57ea2..42a37d253 100644
--- a/acceptance/README.md
+++ b/acceptance/README.md
@@ -9,7 +9,7 @@ To author a test,
 
 The test runner will run script and capture output and compare it with `output.txt` file in the same directory.
 
-In order to write `output.txt` for the first time or overwrite it with the current output, set `TESTS_OUTPUT=OVERWRITE` env var.
+In order to write `output.txt` for the first time or overwrite it with the current output pass -update flag to go test.
 
 The scripts are run with `bash -e` so any errors will be propagated. They are captured in `output.txt` by appending `Exit code: N` line at the end.
 
diff --git a/libs/testdiff/golden.go b/libs/testdiff/golden.go
index b67eb50a9..02213c88a 100644
--- a/libs/testdiff/golden.go
+++ b/libs/testdiff/golden.go
@@ -2,6 +2,7 @@ package testdiff
 
 import (
 	"context"
+	"flag"
 	"fmt"
 	"os"
 	"regexp"
@@ -16,7 +17,11 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-var OverwriteMode = os.Getenv("TESTS_OUTPUT") == "OVERWRITE"
+var OverwriteMode = false
+
+func init() {
+	flag.BoolVar(&OverwriteMode, "update", false, "Overwrite golden files")
+}
 
 func ReadFile(t testutil.TestingT, ctx context.Context, filename string) string {
 	t.Helper()

From b808d4d6f1b860f6286b2988a6226c5d506910b3 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 9 Jan 2025 10:03:03 +0100
Subject: [PATCH 041/247] Add test for overriding list variable (#2099)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Add override for "libraries".
- Remove complexvar - it serves no purpose - we already have map
variable ‘cluster’.
---
 .../bundle/variables/complex/databricks.yml   | 19 +++-------
 .../bundle/variables/complex/out.default.json | 14 -------
 .../bundle/variables/complex/out.dev.json     | 37 ++++---------------
 .../bundle/variables/complex/output.txt       | 14 +++++++
 acceptance/bundle/variables/complex/script    | 10 +++--
 5 files changed, 33 insertions(+), 61 deletions(-)
 create mode 100644 acceptance/bundle/variables/complex/output.txt

diff --git a/acceptance/bundle/variables/complex/databricks.yml b/acceptance/bundle/variables/complex/databricks.yml
index 500f374e3..5dcc30b08 100644
--- a/acceptance/bundle/variables/complex/databricks.yml
+++ b/acceptance/bundle/variables/complex/databricks.yml
@@ -11,6 +11,7 @@ resources:
       - task_key: test
         job_cluster_key: key
         libraries: ${variables.libraries.value}
+        # specific fields of complex variable are referenced:
         task_key: "task with spark version ${var.cluster.spark_version} and jar ${var.libraries[0].jar}"
 
 variables:
@@ -35,14 +36,6 @@ variables:
       - jar: "/path/to/jar"
       - egg: "/path/to/egg"
       - whl: "/path/to/whl"
-  complexvar:
-    type: complex
-    description: "A complex variable"
-    default:
-      key1: "value1"
-      key2: "value2"
-      key3: "value3"
-
 
 targets:
   default:
@@ -51,15 +44,13 @@ targets:
     variables:
       node_type: "Standard_DS3_v3"
       cluster:
+        # complex variables are not merged, so missing variables (policy_id) are not inherited
         spark_version: "14.2.x-scala2.11"
         node_type_id: ${var.node_type}
         num_workers: 4
         spark_conf:
           spark.speculation: false
           spark.databricks.delta.retentionDurationCheck.enabled: false
-      complexvar:
-        type: complex
-        default:
-          key1: "1"
-          key2: "2"
-          key3: "3"
+      libraries:
+        - jar: "/newpath/to/jar"
+        - whl: "/newpath/to/whl"
diff --git a/acceptance/bundle/variables/complex/out.default.json b/acceptance/bundle/variables/complex/out.default.json
index bbdb0f8de..6454562a6 100644
--- a/acceptance/bundle/variables/complex/out.default.json
+++ b/acceptance/bundle/variables/complex/out.default.json
@@ -76,20 +76,6 @@
         "spark_version": "13.2.x-scala2.11"
       }
     },
-    "complexvar": {
-      "default": {
-        "key1": "value1",
-        "key2": "value2",
-        "key3": "value3"
-      },
-      "description": "A complex variable",
-      "type": "complex",
-      "value": {
-        "key1": "value1",
-        "key2": "value2",
-        "key3": "value3"
-      }
-    },
     "libraries": {
       "default": [
         {
diff --git a/acceptance/bundle/variables/complex/out.dev.json b/acceptance/bundle/variables/complex/out.dev.json
index 330518c85..cede5feb2 100644
--- a/acceptance/bundle/variables/complex/out.dev.json
+++ b/acceptance/bundle/variables/complex/out.dev.json
@@ -32,16 +32,13 @@
             "job_cluster_key": "key",
             "libraries": [
               {
-                "jar": "/path/to/jar"
+                "jar": "/newpath/to/jar"
               },
               {
-                "egg": "/path/to/egg"
-              },
-              {
-                "whl": "/path/to/whl"
+                "whl": "/newpath/to/whl"
               }
             ],
-            "task_key": "task with spark version 14.2.x-scala2.11 and jar /path/to/jar"
+            "task_key": "task with spark version 14.2.x-scala2.11 and jar /newpath/to/jar"
           }
         ]
       }
@@ -70,43 +67,23 @@
         "spark_version": "14.2.x-scala2.11"
       }
     },
-    "complexvar": {
-      "default": {
-        "key1": "1",
-        "key2": "2",
-        "key3": "3"
-      },
-      "description": "A complex variable",
-      "type": "complex",
-      "value": {
-        "key1": "1",
-        "key2": "2",
-        "key3": "3"
-      }
-    },
     "libraries": {
       "default": [
         {
-          "jar": "/path/to/jar"
+          "jar": "/newpath/to/jar"
         },
         {
-          "egg": "/path/to/egg"
-        },
-        {
-          "whl": "/path/to/whl"
+          "whl": "/newpath/to/whl"
         }
       ],
       "description": "A libraries definition",
       "type": "complex",
       "value": [
         {
-          "jar": "/path/to/jar"
+          "jar": "/newpath/to/jar"
         },
         {
-          "egg": "/path/to/egg"
-        },
-        {
-          "whl": "/path/to/whl"
+          "whl": "/newpath/to/whl"
         }
       ]
     },
diff --git a/acceptance/bundle/variables/complex/output.txt b/acceptance/bundle/variables/complex/output.txt
new file mode 100644
index 000000000..ce295421f
--- /dev/null
+++ b/acceptance/bundle/variables/complex/output.txt
@@ -0,0 +1,14 @@
+
+>>> $CLI bundle validate -o json
+
+>>> jq .resources.jobs.my_job.tasks[0].task_key out.default.json
+"task with spark version 13.2.x-scala2.11 and jar /path/to/jar"
+
+>>> $CLI bundle validate -o json -t dev
+
+>>> jq .resources.jobs.my_job.tasks[0].task_key out.dev.json
+"task with spark version 14.2.x-scala2.11 and jar /newpath/to/jar"
+policy_id and spark_conf.spark_random fields do not exist in dev target:
+
+>>> jq .resources.jobs.my_job.job_clusters[0].new_cluster.policy_id out.dev.json
+null
diff --git a/acceptance/bundle/variables/complex/script b/acceptance/bundle/variables/complex/script
index 9ee0ab02a..f8b61f18d 100644
--- a/acceptance/bundle/variables/complex/script
+++ b/acceptance/bundle/variables/complex/script
@@ -1,4 +1,8 @@
-$CLI bundle validate -o json | jq '{resources,variables}' > out.default.json
+trace $CLI bundle validate -o json | jq '{resources,variables}' > out.default.json
+trace jq .resources.jobs.my_job.tasks[0].task_key out.default.json | grep "task with spark version 13.2.x-scala2.11 and jar /path/to/jar"
 
-# spark.random and policy_id should be empty in this target:
-$CLI bundle validate -o json -t dev | jq '{resources,variables}' > out.dev.json
+trace $CLI bundle validate -o json -t dev | jq '{resources,variables}' > out.dev.json
+trace jq .resources.jobs.my_job.tasks[0].task_key out.dev.json | grep "task with spark version 14.2.x-scala2.11 and jar /newpath/to/jar"
+
+echo policy_id and spark_conf.spark_random fields do not exist in dev target:
+trace jq .resources.jobs.my_job.job_clusters[0].new_cluster.policy_id out.dev.json | grep null

From 2a4fdd911e2b8ece6a9675bfa409ade9343847cb Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 9 Jan 2025 11:14:47 +0100
Subject: [PATCH 042/247] Add a test showing bug in merge if variables are used
 for keys (#2101)

Since merge happens first, before variable resolution, the two jobs are
seen as different.

I also updated override/job_cluster/script to include more of the
output.
---
 .../bundle/override/job_cluster/output.txt    | 66 ++++++++++-----
 acceptance/bundle/override/job_cluster/script |  4 +-
 .../override/job_cluster_var/databricks.yml   | 37 ++++++++
 .../override/job_cluster_var/output.txt       | 84 +++++++++++++++++++
 .../bundle/override/job_cluster_var/script    |  4 +
 5 files changed, 173 insertions(+), 22 deletions(-)
 create mode 100644 acceptance/bundle/override/job_cluster_var/databricks.yml
 create mode 100644 acceptance/bundle/override/job_cluster_var/output.txt
 create mode 100644 acceptance/bundle/override/job_cluster_var/script

diff --git a/acceptance/bundle/override/job_cluster/output.txt b/acceptance/bundle/override/job_cluster/output.txt
index dc7a5f75b..947d19032 100644
--- a/acceptance/bundle/override/job_cluster/output.txt
+++ b/acceptance/bundle/override/job_cluster/output.txt
@@ -1,30 +1,56 @@
 
 >>> $CLI bundle validate -o json -t development
 {
-  "name": "job",
-  "job_clusters": [
-    {
-      "job_cluster_key": "key",
-      "new_cluster": {
-        "node_type_id": "i3.xlarge",
-        "num_workers": 1,
-        "spark_version": "13.3.x-scala2.12"
+  "foo": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "job_clusters": [
+      {
+        "job_cluster_key": "key",
+        "new_cluster": {
+          "node_type_id": "i3.xlarge",
+          "num_workers": 1,
+          "spark_version": "13.3.x-scala2.12"
+        }
       }
-    }
-  ]
+    ],
+    "name": "job",
+    "permissions": [],
+    "queue": {
+      "enabled": true
+    },
+    "tags": {}
+  }
 }
 
 >>> $CLI bundle validate -o json -t staging
 {
-  "name": "job",
-  "job_clusters": [
-    {
-      "job_cluster_key": "key",
-      "new_cluster": {
-        "node_type_id": "i3.2xlarge",
-        "num_workers": 4,
-        "spark_version": "13.3.x-scala2.12"
+  "foo": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "job_clusters": [
+      {
+        "job_cluster_key": "key",
+        "new_cluster": {
+          "node_type_id": "i3.2xlarge",
+          "num_workers": 4,
+          "spark_version": "13.3.x-scala2.12"
+        }
       }
-    }
-  ]
+    ],
+    "name": "job",
+    "permissions": [],
+    "queue": {
+      "enabled": true
+    },
+    "tags": {}
+  }
 }
diff --git a/acceptance/bundle/override/job_cluster/script b/acceptance/bundle/override/job_cluster/script
index 3f6827bb7..4a26c433a 100644
--- a/acceptance/bundle/override/job_cluster/script
+++ b/acceptance/bundle/override/job_cluster/script
@@ -1,2 +1,2 @@
-trace $CLI bundle validate -o json -t development | jq '.resources.jobs.foo | {name,job_clusters}'
-trace $CLI bundle validate -o json -t staging | jq '.resources.jobs.foo | {name,job_clusters}'
+trace $CLI bundle validate -o json -t development | jq '.resources.jobs'
+trace $CLI bundle validate -o json -t staging | jq '.resources.jobs'
diff --git a/acceptance/bundle/override/job_cluster_var/databricks.yml b/acceptance/bundle/override/job_cluster_var/databricks.yml
new file mode 100644
index 000000000..546cc2d8a
--- /dev/null
+++ b/acceptance/bundle/override/job_cluster_var/databricks.yml
@@ -0,0 +1,37 @@
+bundle:
+  name: override_job_cluster
+
+variables:
+  mykey:
+    default: key
+
+resources:
+  jobs:
+    foo:
+      name: job
+      job_clusters:
+        - job_cluster_key: key
+          new_cluster:
+            spark_version: 13.3.x-scala2.12
+
+targets:
+  development:
+    resources:
+      jobs:
+        foo:
+          job_clusters:
+            # This does not work because merging is done before resolution
+            - job_cluster_key: "${var.mykey}"
+              new_cluster:
+                node_type_id: i3.xlarge
+                num_workers: 1
+
+  staging:
+    resources:
+      jobs:
+        foo:
+          job_clusters:
+            - job_cluster_key: "${var.mykey}"
+              new_cluster:
+                node_type_id: i3.2xlarge
+                num_workers: 4
diff --git a/acceptance/bundle/override/job_cluster_var/output.txt b/acceptance/bundle/override/job_cluster_var/output.txt
new file mode 100644
index 000000000..dee2a3b5b
--- /dev/null
+++ b/acceptance/bundle/override/job_cluster_var/output.txt
@@ -0,0 +1,84 @@
+
+>>> $CLI bundle validate -o json -t development
+{
+  "foo": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "job_clusters": [
+      {
+        "job_cluster_key": "key",
+        "new_cluster": {
+          "spark_version": "13.3.x-scala2.12"
+        }
+      },
+      {
+        "job_cluster_key": "key",
+        "new_cluster": {
+          "node_type_id": "i3.xlarge",
+          "num_workers": 1
+        }
+      }
+    ],
+    "name": "job",
+    "permissions": [],
+    "queue": {
+      "enabled": true
+    },
+    "tags": {}
+  }
+}
+
+>>> $CLI bundle validate -t development
+Name: override_job_cluster
+Target: development
+Workspace:
+  User: tester@databricks.com
+  Path: /Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development
+
+Validation OK!
+
+>>> $CLI bundle validate -o json -t staging
+{
+  "foo": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "job_clusters": [
+      {
+        "job_cluster_key": "key",
+        "new_cluster": {
+          "spark_version": "13.3.x-scala2.12"
+        }
+      },
+      {
+        "job_cluster_key": "key",
+        "new_cluster": {
+          "node_type_id": "i3.2xlarge",
+          "num_workers": 4
+        }
+      }
+    ],
+    "name": "job",
+    "permissions": [],
+    "queue": {
+      "enabled": true
+    },
+    "tags": {}
+  }
+}
+
+>>> $CLI bundle validate -t staging
+Name: override_job_cluster
+Target: staging
+Workspace:
+  User: tester@databricks.com
+  Path: /Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging
+
+Validation OK!
diff --git a/acceptance/bundle/override/job_cluster_var/script b/acceptance/bundle/override/job_cluster_var/script
new file mode 100644
index 000000000..1cf373828
--- /dev/null
+++ b/acceptance/bundle/override/job_cluster_var/script
@@ -0,0 +1,4 @@
+trace $CLI bundle validate -o json -t development | jq '.resources.jobs'
+trace $CLI bundle validate -t development
+trace $CLI bundle validate -o json -t staging | jq '.resources.jobs'
+trace $CLI bundle validate -t staging

From 3b3ede6e31181396c46a6ddb86299566e23b75fe Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Thu, 9 Jan 2025 12:21:30 +0100
Subject: [PATCH 043/247] Update runner for the publish-winget job (#2105)

## Changes

This action uses a token to access the release artifacts and, as such,
needs to execute on the runner that's on the allowlist.

Related PRs:
* #2098
* #2077
---
 .github/workflows/publish-winget.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/publish-winget.yml b/.github/workflows/publish-winget.yml
index 19603e669..dd3aebcbd 100644
--- a/.github/workflows/publish-winget.yml
+++ b/.github/workflows/publish-winget.yml
@@ -5,8 +5,12 @@ on:
 
 jobs:
   publish-to-winget-pkgs:
-    runs-on: windows-latest
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: windows-server-latest
+
     environment: release
+
     steps:
       - uses: vedantmgoyal2009/winget-releaser@93fd8b606a1672ec3e5c6c3bb19426be68d1a8b0 # https://github.com/vedantmgoyal2009/winget-releaser/releases/tag/v2
         with:

From 4b67e9f33611089f177ad68431ee2070a195261b Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Thu, 9 Jan 2025 13:07:29 +0100
Subject: [PATCH 044/247] Pass tag to release as input to publish-winget
 workflow (#2107)

## Changes

This workflow only worked if it was triggered on the tag to publish
itself. This means it is not possible to release a version if the
workflow configuration at that tag is broken (as is the case for
v0.238.0 because of #2105).

This change adds a "tag" input that can be set when manually triggering
the workflow.

## Tests

* Succesful run with this change:
https://github.com/databricks/cli/actions/runs/12689281843
* Pull request that the run created:
https://github.com/microsoft/winget-pkgs/pull/209220
---
 .github/workflows/publish-winget.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/publish-winget.yml b/.github/workflows/publish-winget.yml
index dd3aebcbd..267077102 100644
--- a/.github/workflows/publish-winget.yml
+++ b/.github/workflows/publish-winget.yml
@@ -2,6 +2,10 @@ name: publish-winget
 
 on:
   workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Tag to publish'
+        default: ''
 
 jobs:
   publish-to-winget-pkgs:
@@ -18,3 +22,7 @@ jobs:
           installers-regex: 'windows_.*-signed\.zip$' # Only signed Windows releases
           token: ${{ secrets.ENG_DEV_ECOSYSTEM_BOT_TOKEN }}
           fork-user: eng-dev-ecosystem-bot
+
+          # Use the tag from the input, or the ref name if the input is not provided.
+          # The ref name is equal to the tag name when this workflow is triggered by the "sign-cli" command.
+          release-tag: ${{ inputs.tag || github.ref_name }}

From a0455bcaefa488ca736b0baba0573da7dceeb30d Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 9 Jan 2025 16:21:24 +0100
Subject: [PATCH 045/247] Migrate bundle/tests/undefined_resources_test.go to
 acceptance test (#2106)

Add sort_blocks.py helper to deal with non-determinism.
---
 acceptance/acceptance_test.go                 | 12 +++--
 acceptance/bin/sort_blocks.py                 | 21 ++++++++
 .../undefined_resources/databricks.yml        |  0
 .../bundle/undefined_resources/output.txt     | 19 +++++++
 acceptance/bundle/undefined_resources/script  |  2 +
 bundle/tests/undefined_resources_test.go      | 50 -------------------
 6 files changed, 50 insertions(+), 54 deletions(-)
 create mode 100755 acceptance/bin/sort_blocks.py
 rename {bundle/tests => acceptance/bundle}/undefined_resources/databricks.yml (100%)
 create mode 100644 acceptance/bundle/undefined_resources/output.txt
 create mode 100644 acceptance/bundle/undefined_resources/script
 delete mode 100644 bundle/tests/undefined_resources_test.go

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 759c0aeca..033f26dfb 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -35,10 +35,16 @@ var Scripts = map[string]bool{
 }
 
 func TestAccept(t *testing.T) {
-	execPath := BuildCLI(t)
+	cwd, err := os.Getwd()
+	require.NoError(t, err)
+
+	execPath := BuildCLI(t, cwd)
 	// $CLI is what test scripts are using
 	t.Setenv("CLI", execPath)
 
+	// Make helper scripts available
+	t.Setenv("PATH", fmt.Sprintf("%s%c%s", filepath.Join(cwd, "bin"), os.PathListSeparator, os.Getenv("PATH")))
+
 	server := StartServer(t)
 	AddHandlers(server)
 	// Redirect API access to local server:
@@ -199,9 +205,7 @@ func readMergedScriptContents(t *testing.T, dir string) string {
 	return strings.Join(prepares, "\n")
 }
 
-func BuildCLI(t *testing.T) string {
-	cwd, err := os.Getwd()
-	require.NoError(t, err)
+func BuildCLI(t *testing.T, cwd string) string {
 	execPath := filepath.Join(cwd, "build", "databricks")
 	if runtime.GOOS == "windows" {
 		execPath += ".exe"
diff --git a/acceptance/bin/sort_blocks.py b/acceptance/bin/sort_blocks.py
new file mode 100755
index 000000000..f50c6f50f
--- /dev/null
+++ b/acceptance/bin/sort_blocks.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+"""
+Helper to sort blocks in text file. A block is a set of lines separated from others by empty line.
+
+This is to workaround non-determinism in the output.
+"""
+import sys
+
+blocks = []
+
+for line in sys.stdin:
+    if not line.strip():
+        if blocks and blocks[-1]:
+            blocks.append('')
+        continue
+    if not blocks:
+        blocks.append('')
+    blocks[-1] += line
+
+blocks.sort()
+print("\n".join(blocks))
diff --git a/bundle/tests/undefined_resources/databricks.yml b/acceptance/bundle/undefined_resources/databricks.yml
similarity index 100%
rename from bundle/tests/undefined_resources/databricks.yml
rename to acceptance/bundle/undefined_resources/databricks.yml
diff --git a/acceptance/bundle/undefined_resources/output.txt b/acceptance/bundle/undefined_resources/output.txt
new file mode 100644
index 000000000..29b51bc1a
--- /dev/null
+++ b/acceptance/bundle/undefined_resources/output.txt
@@ -0,0 +1,19 @@
+Error: experiment undefined-experiment is not defined
+  at resources.experiments.undefined-experiment
+  in databricks.yml:11:26
+
+Error: job undefined-job is not defined
+  at resources.jobs.undefined-job
+  in databricks.yml:6:19
+
+Error: pipeline undefined-pipeline is not defined
+  at resources.pipelines.undefined-pipeline
+  in databricks.yml:14:24
+
+Found 3 errors
+
+Name: undefined-job
+Target: default
+
+
+Exit code: 1
diff --git a/acceptance/bundle/undefined_resources/script b/acceptance/bundle/undefined_resources/script
new file mode 100644
index 000000000..10a3c485a
--- /dev/null
+++ b/acceptance/bundle/undefined_resources/script
@@ -0,0 +1,2 @@
+# We need sort_blocks.py because the order of diagnostics is currently randomized
+$CLI bundle validate 2>&1 | sort_blocks.py
diff --git a/bundle/tests/undefined_resources_test.go b/bundle/tests/undefined_resources_test.go
deleted file mode 100644
index 3dbacbc25..000000000
--- a/bundle/tests/undefined_resources_test.go
+++ /dev/null
@@ -1,50 +0,0 @@
-package config_tests
-
-import (
-	"context"
-	"path/filepath"
-	"testing"
-
-	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/bundle/config/validate"
-	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/dyn"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestUndefinedResourcesLoadWithError(t *testing.T) {
-	b := load(t, "./undefined_resources")
-	diags := bundle.Apply(context.Background(), b, validate.AllResourcesHaveValues())
-
-	assert.Len(t, diags, 3)
-	assert.Contains(t, diags, diag.Diagnostic{
-		Severity: diag.Error,
-		Summary:  "job undefined-job is not defined",
-		Locations: []dyn.Location{{
-			File:   filepath.FromSlash("undefined_resources/databricks.yml"),
-			Line:   6,
-			Column: 19,
-		}},
-		Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.undefined-job")},
-	})
-	assert.Contains(t, diags, diag.Diagnostic{
-		Severity: diag.Error,
-		Summary:  "experiment undefined-experiment is not defined",
-		Locations: []dyn.Location{{
-			File:   filepath.FromSlash("undefined_resources/databricks.yml"),
-			Line:   11,
-			Column: 26,
-		}},
-		Paths: []dyn.Path{dyn.MustPathFromString("resources.experiments.undefined-experiment")},
-	})
-	assert.Contains(t, diags, diag.Diagnostic{
-		Severity: diag.Error,
-		Summary:  "pipeline undefined-pipeline is not defined",
-		Locations: []dyn.Location{{
-			File:   filepath.FromSlash("undefined_resources/databricks.yml"),
-			Line:   14,
-			Column: 24,
-		}},
-		Paths: []dyn.Path{dyn.MustPathFromString("resources.pipelines.undefined-pipeline")},
-	})
-}

From b0c1c236307b19a7fa8df472756e48c3e8f47897 Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Thu, 9 Jan 2025 23:49:34 +0530
Subject: [PATCH 046/247] Add `uuid` to builtin templates (#2088)

## Changes
This is useful to track telemetry associated with the templates and can
later be useful for functional usecases as well. Mlops stacks does the
same here: https://github.com/databricks/mlops-stacks/pull/185

## Tests
Existing tests.
---
 integration/bundle/testdata/default_python/bundle_summary.txt  | 3 ++-
 .../dbt-sql/template/{{.project_name}}/databricks.yml.tmpl     | 1 +
 .../template/{{.project_name}}/databricks.yml.tmpl             | 1 +
 .../default-sql/template/{{.project_name}}/databricks.yml.tmpl | 1 +
 4 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/integration/bundle/testdata/default_python/bundle_summary.txt b/integration/bundle/testdata/default_python/bundle_summary.txt
index 3143d729c..a0bcfdbc8 100644
--- a/integration/bundle/testdata/default_python/bundle_summary.txt
+++ b/integration/bundle/testdata/default_python/bundle_summary.txt
@@ -15,7 +15,8 @@
       "lock": {
         "enabled": false
       }
-    }
+    },
+    "uuid": "<UUID>"
   },
   "include": [
     "resources/project_name_$UNIQUE_PRJ.job.yml",
diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl
index 5594749a9..ba336f6a1 100644
--- a/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl
+++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl
@@ -3,6 +3,7 @@
 # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
 bundle:
   name: {{.project_name}}
+  uuid: {{bundle_uuid}}
 
 include:
   - resources/*.yml
diff --git a/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl
index c42b822a8..4d052e38e 100644
--- a/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl
+++ b/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl
@@ -2,6 +2,7 @@
 # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
 bundle:
   name: {{.project_name}}
+  uuid: {{bundle_uuid}}
 
 include:
   - resources/*.yml
diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl
index 51d03e99a..84e07df17 100644
--- a/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl
+++ b/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl
@@ -2,6 +2,7 @@
 # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
 bundle:
   name: {{.project_name}}
+  uuid: {{bundle_uuid}}
 
 include:
   - resources/*.yml

From 6d3b4159bd5911c22b33c65ae245f63e06462d85 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 10 Jan 2025 09:51:59 +0100
Subject: [PATCH 047/247] Log warnings to stderr for "bundle validate -o json"
 (#2109)

## Changes
Previously diagnostics were not seen in JSON output mode. This change
prints them to stderr.

This also fixes acceptance tests to preprocess all output with
s/execPath/$CLI/ not just output.txt.

## Tests
Existing acceptance tests. In one case I've added non-json command to
check that they match in output.
---
 acceptance/acceptance_test.go                 | 13 +++++++----
 .../job_tasks/out.development.stderr.txt      |  6 +++++
 .../bundle/override/job_tasks/output.txt      | 19 +++++++++++----
 acceptance/bundle/override/job_tasks/script   |  3 ++-
 .../override/merge-string-map/output.txt      |  4 ++++
 cmd/bundle/validate.go                        | 23 +++++++++++++++----
 6 files changed, 54 insertions(+), 14 deletions(-)
 create mode 100644 acceptance/bundle/override/job_tasks/out.development.stderr.txt

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 033f26dfb..b9fb219dc 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -55,12 +55,15 @@ func TestAccept(t *testing.T) {
 	// Do not read user's ~/.databrickscfg
 	t.Setenv(env.HomeEnvVar(), homeDir)
 
+	repls := testdiff.ReplacementsContext{}
+	repls.Set(execPath, "$CLI")
+
 	testDirs := getTests(t)
 	require.NotEmpty(t, testDirs)
 	for _, dir := range testDirs {
 		t.Run(dir, func(t *testing.T) {
 			t.Parallel()
-			runTest(t, dir)
+			runTest(t, dir, repls)
 		})
 	}
 }
@@ -85,7 +88,7 @@ func getTests(t *testing.T) []string {
 	return testDirs
 }
 
-func runTest(t *testing.T, dir string) {
+func runTest(t *testing.T, dir string, repls testdiff.ReplacementsContext) {
 	var tmpDir string
 	var err error
 	if KeepTmp {
@@ -112,7 +115,7 @@ func runTest(t *testing.T, dir string) {
 	outB, err := cmd.CombinedOutput()
 
 	out := formatOutput(string(outB), err)
-	out = strings.ReplaceAll(out, os.Getenv("CLI"), "$CLI")
+	out = repls.Replace(out)
 	doComparison(t, filepath.Join(dir, "output.txt"), "script output", out)
 
 	for key := range outputs {
@@ -131,7 +134,8 @@ func runTest(t *testing.T, dir string) {
 			continue
 		}
 		pathExpected := filepath.Join(dir, key)
-		doComparison(t, pathExpected, pathNew, string(newValBytes))
+		newVal := repls.Replace(string(newValBytes))
+		doComparison(t, pathExpected, pathNew, newVal)
 	}
 
 	// Make sure there are not unaccounted for new files
@@ -152,6 +156,7 @@ func runTest(t *testing.T, dir string) {
 			// Show the contents & support overwrite mode for it:
 			pathNew := filepath.Join(tmpDir, name)
 			newVal := testutil.ReadFile(t, pathNew)
+			newVal = repls.Replace(newVal)
 			doComparison(t, filepath.Join(dir, name), filepath.Join(tmpDir, name), newVal)
 		}
 	}
diff --git a/acceptance/bundle/override/job_tasks/out.development.stderr.txt b/acceptance/bundle/override/job_tasks/out.development.stderr.txt
new file mode 100644
index 000000000..7b6fef0cc
--- /dev/null
+++ b/acceptance/bundle/override/job_tasks/out.development.stderr.txt
@@ -0,0 +1,6 @@
+
+>>> errcode $CLI bundle validate -o json -t development
+Error: file ./test1.py not found
+
+
+Exit code: 1
diff --git a/acceptance/bundle/override/job_tasks/output.txt b/acceptance/bundle/override/job_tasks/output.txt
index 0d561291e..0bb0b1812 100644
--- a/acceptance/bundle/override/job_tasks/output.txt
+++ b/acceptance/bundle/override/job_tasks/output.txt
@@ -1,8 +1,3 @@
-
->>> errcode $CLI bundle validate -o json -t development
-Error: file ./test1.py not found
-
-Exit code: 1
 {
   "name": "job",
   "queue": {
@@ -36,6 +31,7 @@ Exit code: 1
 >>> errcode $CLI bundle validate -o json -t staging
 Error: file ./test1.py not found
 
+
 Exit code: 1
 {
   "name": "job",
@@ -66,3 +62,16 @@ Exit code: 1
     }
   ]
 }
+
+>>> errcode $CLI bundle validate -t staging
+Error: file ./test1.py not found
+
+Name: override_job_tasks
+Target: staging
+Workspace:
+  User: tester@databricks.com
+  Path: /Workspace/Users/tester@databricks.com/.bundle/override_job_tasks/staging
+
+Found 1 error
+
+Exit code: 1
diff --git a/acceptance/bundle/override/job_tasks/script b/acceptance/bundle/override/job_tasks/script
index 4e0869857..f41729c1e 100644
--- a/acceptance/bundle/override/job_tasks/script
+++ b/acceptance/bundle/override/job_tasks/script
@@ -1,2 +1,3 @@
-trace errcode $CLI bundle validate -o json -t development | jq .resources.jobs.foo
+trace errcode $CLI bundle validate -o json -t development 2> out.development.stderr.txt | jq .resources.jobs.foo
 trace errcode $CLI bundle validate -o json -t staging | jq .resources.jobs.foo
+trace errcode $CLI bundle validate -t staging
diff --git a/acceptance/bundle/override/merge-string-map/output.txt b/acceptance/bundle/override/merge-string-map/output.txt
index e1bd7dfb4..986da8174 100644
--- a/acceptance/bundle/override/merge-string-map/output.txt
+++ b/acceptance/bundle/override/merge-string-map/output.txt
@@ -1,5 +1,9 @@
 
 >>> $CLI bundle validate -o json -t dev
+Warning: expected map, found string
+  at resources.clusters.my_cluster
+  in databricks.yml:6:17
+
 {
   "clusters": {
     "my_cluster": {
diff --git a/cmd/bundle/validate.go b/cmd/bundle/validate.go
index daeb7426d..41fa87f30 100644
--- a/cmd/bundle/validate.go
+++ b/cmd/bundle/validate.go
@@ -11,18 +11,17 @@ import (
 	"github.com/databricks/cli/bundle/render"
 	"github.com/databricks/cli/cmd/bundle/utils"
 	"github.com/databricks/cli/cmd/root"
-	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/flags"
 	"github.com/spf13/cobra"
 )
 
-func renderJsonOutput(cmd *cobra.Command, b *bundle.Bundle, diags diag.Diagnostics) error {
+func renderJsonOutput(cmd *cobra.Command, b *bundle.Bundle) error {
 	buf, err := json.MarshalIndent(b.Config.Value().AsAny(), "", "  ")
 	if err != nil {
 		return err
 	}
 	_, _ = cmd.OutOrStdout().Write(buf)
-	return diags.Error()
+	return nil
 }
 
 func newValidateCommand() *cobra.Command {
@@ -66,7 +65,23 @@ func newValidateCommand() *cobra.Command {
 
 			return nil
 		case flags.OutputJSON:
-			return renderJsonOutput(cmd, b, diags)
+			renderOpts := render.RenderOptions{RenderSummaryTable: false}
+			err1 := render.RenderDiagnostics(cmd.ErrOrStderr(), b, diags, renderOpts)
+			err2 := renderJsonOutput(cmd, b)
+
+			if err2 != nil {
+				return err2
+			}
+
+			if err1 != nil {
+				return err1
+			}
+
+			if diags.HasError() {
+				return root.ErrAlreadyPrinted
+			}
+
+			return nil
 		default:
 			return fmt.Errorf("unknown output type %s", root.OutputType(cmd))
 		}

From f2c4cae9f1e05aaf24bb02efa3f60df51cd7ac50 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 10 Jan 2025 10:32:39 +0100
Subject: [PATCH 048/247] Increase close-after-stale from 7 to 30 days (#2111)

Giving 7 days to react before closing is too aggressive, IMO. Changed it
to 30.

Also changed 'stale' label from 30d to 60d.

Also removed dry-run setting, it does not appear to do anything.
---
 .github/workflows/close-stale-issues.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/close-stale-issues.yml b/.github/workflows/close-stale-issues.yml
index 273b89a9c..7bf754319 100644
--- a/.github/workflows/close-stale-issues.yml
+++ b/.github/workflows/close-stale-issues.yml
@@ -31,10 +31,8 @@ jobs:
           exempt-pr-labels: No Autoclose
 
           # Issue timing
-          days-before-stale: 30
-          days-before-close: 7
+          days-before-stale: 60
+          days-before-close: 30
 
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           loglevel: DEBUG
-          # TODO: Remove dry-run after merge when confirmed it works correctly
-          dry-run: true

From 72e833a8975529b28e0c2396eb3c41c8dfb94e7c Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 10 Jan 2025 10:39:00 +0100
Subject: [PATCH 049/247] Configure dependabot to check for new github-actions
 (#2112)

---
 .github/dependabot.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index f1b219b47..e7d7ad6b6 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -4,3 +4,7 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"

From 75cd582021a4c0130f888f58fc7f33687d44a642 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 10 Jan 2025 10:49:33 +0100
Subject: [PATCH 050/247] Remove lint.sh; re-add 'make fmt' (#2113)

See Makefile for explanation on difference between 'make fmt' and 'make
lint'.

I also removed lint.sh. Original motivation was to use it in aider, but
it's not a good fit there, because aider passes filenames and it does
not work well with most golang linters which requires whole packages to
work.

Follow up to #2062, #2056, #2051.
---
 Makefile | 10 ++++++++--
 lint.sh  | 14 --------------
 2 files changed, 8 insertions(+), 16 deletions(-)
 delete mode 100755 lint.sh

diff --git a/Makefile b/Makefile
index 40eef9f31..2c84d88ba 100644
--- a/Makefile
+++ b/Makefile
@@ -5,11 +5,17 @@ PACKAGES=./acceptance/... ./libs/... ./internal/... ./cmd/... ./bundle/... .
 GOTESTSUM_FORMAT ?= pkgname-and-test-fails
 
 lint:
-	./lint.sh ./...
+	golangci-lint run --fix
 
 lintcheck:
 	golangci-lint run ./...
 
+# Note 'make lint' will do formatting as well. However, if there are compilation errors,
+# formatting/goimports will not be applied by 'make lint'. However, it will be applied by 'make fmt'.
+# If you need to ensure that formatting & imports are always fixed, do "make fmt lint"
+fmt:
+	golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix ./...
+
 test:
 	gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped -- ${PACKAGES}
 
@@ -39,4 +45,4 @@ integration:
 integration-short:
 	$(INTEGRATION) -short
 
-.PHONY: lint lintcheck test cover showcover build snapshot vendor schema integration integration-short
+.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short
diff --git a/lint.sh b/lint.sh
deleted file mode 100755
index 1f881eaf7..000000000
--- a/lint.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-set -uo pipefail
-# With golangci-lint, if there are any compliation issues, then formatters' autofix won't be applied.
-# https://github.com/golangci/golangci-lint/issues/5257
-
-golangci-lint run --fix "$@"
-lint_exit_code=$?
-
-if [ $lint_exit_code -ne 0 ]; then
-    # These linters work in presence of compilation issues when run alone, so let's get these fixes at least.
-    golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix "$@"
-fi
-
-exit $lint_exit_code

From 99cd3fe1846d16e0b67520e390260c46badd778e Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 10 Jan 2025 10:49:57 +0100
Subject: [PATCH 051/247] Bump golangci-lint version to v1.63.4 from v1.63.1
 (#2114)

---
 .github/workflows/push.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
index b71b23c4b..42245b14f 100644
--- a/.github/workflows/push.yml
+++ b/.github/workflows/push.yml
@@ -97,7 +97,7 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v6
         with:
-          version: v1.63.1
+          version: v1.63.4
           args: --timeout=15m
 
   validate-bundle-schema:

From dc3a157fdc3d800dbe0bcd7044e95eabb85275a3 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Fri, 10 Jan 2025 11:16:53 +0100
Subject: [PATCH 052/247] Remove cleanup in testcli package (#2108)

## Changes

The main CLI entry point used to be a global variable, and the global
state had to be cleaned up after every test run. This hasn't been the
case for a while, and instead, the CLI is initialized in a function
call. State accumulated by a single CLI "instance" can no longer leak
into other instances, so we no longer have to perform cleanup.

## Tests

Existing tests pass.
---
 internal/testcli/runner.go | 48 --------------------------------------
 1 file changed, 48 deletions(-)

diff --git a/internal/testcli/runner.go b/internal/testcli/runner.go
index 52decad2c..d32fa3947 100644
--- a/internal/testcli/runner.go
+++ b/internal/testcli/runner.go
@@ -6,13 +6,10 @@ import (
 	"context"
 	"encoding/json"
 	"io"
-	"reflect"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
 	"github.com/stretchr/testify/require"
 
 	"github.com/databricks/cli/cmd"
@@ -68,39 +65,6 @@ func consumeLines(ctx context.Context, wg *sync.WaitGroup, r io.Reader) <-chan s
 	return ch
 }
 
-func (r *Runner) registerFlagCleanup(c *cobra.Command) {
-	r.Helper()
-	// Find target command that will be run. Example: if the command run is `databricks fs cp`,
-	// target command corresponds to `cp`
-	targetCmd, _, err := c.Find(r.args)
-	if err != nil && strings.HasPrefix(err.Error(), "unknown command") {
-		// even if command is unknown, we can proceed
-		require.NotNil(r, targetCmd)
-	} else {
-		require.NoError(r, err)
-	}
-
-	// Force initialization of default flags.
-	// These are initialized by cobra at execution time and would otherwise
-	// not be cleaned up by the cleanup function below.
-	targetCmd.InitDefaultHelpFlag()
-	targetCmd.InitDefaultVersionFlag()
-
-	// Restore flag values to their original value on test completion.
-	targetCmd.Flags().VisitAll(func(f *pflag.Flag) {
-		v := reflect.ValueOf(f.Value)
-		if v.Kind() == reflect.Ptr {
-			v = v.Elem()
-		}
-		// Store copy of the current flag value.
-		reset := reflect.New(v.Type()).Elem()
-		reset.Set(v)
-		r.Cleanup(func() {
-			v.Set(reset)
-		})
-	})
-}
-
 // Like [Runner.Eventually], but more specific
 func (r *Runner) WaitForTextPrinted(text string, timeout time.Duration) {
 	r.Eventually(func() bool {
@@ -159,12 +123,6 @@ func (r *Runner) RunBackground() {
 		cli.SetIn(r.stdinR)
 	}
 
-	// Register cleanup function to restore flags to their original values
-	// once test has been executed. This is needed because flag values reside
-	// in a global singleton data-structure, and thus subsequent tests might
-	// otherwise interfere with each other
-	r.registerFlagCleanup(cli)
-
 	errch := make(chan error)
 	ctx, cancel := context.WithCancel(ctx)
 
@@ -208,12 +166,6 @@ func (r *Runner) RunBackground() {
 			}
 		}
 
-		// Reset context on command for the next test.
-		// These commands are globals so we have to clean up to the best of our ability after each run.
-		// See https://github.com/spf13/cobra/blob/a6f198b635c4b18fff81930c40d464904e55b161/command.go#L1062-L1066
-		//nolint:staticcheck  // cobra sets the context and doesn't clear it
-		cli.SetContext(nil)
-
 		// Make caller aware of error.
 		errch <- err
 		close(errch)

From d525ff67be87cf1489211ff6edf781de6c92ac43 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 10 Jan 2025 17:49:05 +0100
Subject: [PATCH 053/247] Bump astral-sh/setup-uv from 4 to 5 (#2116)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bumps [astral-sh/setup-uv](https://github.com/astral-sh/setup-uv) from 4
to 5.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/astral-sh/setup-uv/releases">astral-sh/setup-uv's
releases</a>.</em></p>
<blockquote>
<h2>v5.0.0 🎄 Merry Christmas - Help fastly and users by default</h2>
<h2>Changes</h2>
<p>This christmans 🎄 release is a bit early bit still full of presents 🎁
Since we are changing some of the defaults this can lead to breaking
changes, thus the major version increase.</p>
<p>Here are the highlights:</p>
<h3><a
href="https://redirect.github.com/astral-sh/setup-uv/pull/193">Default
to enable-cache: true on GitHub hosted runners</a></h3>
<p>Did you know that that Fastly, the company hosting PyPI,
theoretically has to pay $12.5 million per month and so far have served
more than 2.41 <strong>exabytes</strong> of data?
<img
src="https://github.com/user-attachments/assets/f2f6cb3f-68f6-4e37-abb1-d3bf1f278533"
alt="image" /></p>
<p>This is why <a
href="https://redirect.github.com/astral-sh/setup-uv/issues/54">they
asked us</a> to turn on caching by default. After weighting the pros and
cons we decided to automatically upload the cache to the GitHub Actions
cache when running on GitHub hosted runners. You can still disable that
with <code>enable-cache: false</code>.</p>
<p>I remember when I first got into actions and didn't understand all
the magic. I was baffled that some actions did something behind the
scenes to make everything faster. I hope with this change we help a lot
of users who are don't want to or are afraid to understand what
<code>enable-cache</code> does.</p>
<h3><a
href="https://redirect.github.com/astral-sh/setup-uv/pull/185">Add
**/requirements*.txt to default cache-dependency-glob</a></h3>
<p>If caching is enabled we automatically searched for a
<code>uv.lock</code> file and when this changed we knew we had to
refresh the cache. A lot of projects don't use this but rather the good
old <code>requirements.txt</code>. We now automatically search for both
<code>uv.lock</code>and <code>requirements*.txt</code> (this means also
<code>requirements-test.txt</code>, <code>requirements-dev.txt</code>,
...) files.
You can change this with <code>cache-dependency-glob</code></p>
<h3><a
href="https://redirect.github.com/astral-sh/setup-uv/pull/194">Auto
activate venv when python-version is set</a></h3>
<p>Some workflows install packages on the fly. This automatically works
when using a python version that is already present on the runner. But
if uv installs the version, e.g. because it is a free-threaded version
or an old one, it is a <a
href="https://astral.sh/blog/python-build-standalone">standalone-build</a>
and installing packages &quot;into the system&quot; is not possible.</p>
<p>We now automatically create a new virtual environment with <code>uv
venv</code> and activate it for the rest of the workflow if
<code>python-version</code> is used. This means you can now do</p>
<pre lang="yaml"><code>- name: Install uv
  uses: astral-sh/setup-uv@auto-environment
  with:
    python-version: 3.13t
- run: uv pip install -i
https://pypi.anaconda.org/scientific-python-nightly-wheels/simple cython
</code></pre>
<h2>🚨 Breaking changes</h2>
<ul>
<li>Default to enable-cache: true on GitHub hosted runners <a
href="https://github.com/eifinger"><code>@​eifinger</code></a> (<a
href="https://redirect.github.com/astral-sh/setup-uv/issues/193">#193</a>)</li>
<li>Add **/requirements*.txt to default cache-dependency-glob <a
href="https://github.com/eifinger"><code>@​eifinger</code></a> (<a
href="https://redirect.github.com/astral-sh/setup-uv/issues/185">#185</a>)</li>
</ul>
<h2>🐛 Bug fixes</h2>
<ul>
<li>Always use api.github.com <a
href="https://github.com/eifinger"><code>@​eifinger</code></a> (<a
href="https://redirect.github.com/astral-sh/setup-uv/issues/191">#191</a>)</li>
</ul>
<h2>🚀 Enhancements</h2>
<ul>
<li>Auto activate venv when python-version is set <a
href="https://github.com/eifinger"><code>@​eifinger</code></a> (<a
href="https://redirect.github.com/astral-sh/setup-uv/issues/194">#194</a>)</li>
<li>Add python version to cache key <a
href="https://github.com/eifinger"><code>@​eifinger</code></a> (<a
href="https://redirect.github.com/astral-sh/setup-uv/issues/187">#187</a>)</li>
</ul>
<!-- raw HTML omitted -->
</blockquote>
<p>... (truncated)</p>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/astral-sh/setup-uv/commit/887a942a15af3a7626099df99e897a18d9e5ab3a"><code>887a942</code></a>
Set VIRTUAL_ENV to .venv instead of .venv/bin (<a
href="https://redirect.github.com/astral-sh/setup-uv/issues/210">#210</a>)</li>
<li><a
href="https://github.com/astral-sh/setup-uv/commit/d174a24c07041141748ff40b1807fb773147fbb5"><code>d174a24</code></a>
Align use of <code>actions/setup-python</code> with uv docu (<a
href="https://redirect.github.com/astral-sh/setup-uv/issues/207">#207</a>)</li>
<li><a
href="https://github.com/astral-sh/setup-uv/commit/12c852e6bac2105511e2d07df46ebf64bec0e25c"><code>12c852e</code></a>
Remove uv version from cache key (<a
href="https://redirect.github.com/astral-sh/setup-uv/issues/206">#206</a>)</li>
<li><a
href="https://github.com/astral-sh/setup-uv/commit/180f8b44399608a850e1db031fa65c77746566d3"><code>180f8b4</code></a>
Fix wrong cacheDependencyPathHash (<a
href="https://redirect.github.com/astral-sh/setup-uv/issues/201">#201</a>)</li>
<li><a
href="https://github.com/astral-sh/setup-uv/commit/e3fb95a68959a9be3d650989dc1d43a9a324447a"><code>e3fb95a</code></a>
Warn instead of fail for no-dependency-glob (<a
href="https://redirect.github.com/astral-sh/setup-uv/issues/200">#200</a>)</li>
<li><a
href="https://github.com/astral-sh/setup-uv/commit/2af22b5b2dcfc0729ee842c635f300f1fc5a9e9a"><code>2af22b5</code></a>
chore: update known checksums for 0.5.11 (<a
href="https://redirect.github.com/astral-sh/setup-uv/issues/198">#198</a>)</li>
<li><a
href="https://github.com/astral-sh/setup-uv/commit/dd578776bbb50c3f5b3de9e6379065aa0077c56f"><code>dd57877</code></a>
Auto activate venv when python-version is set (<a
href="https://redirect.github.com/astral-sh/setup-uv/issues/194">#194</a>)</li>
<li><a
href="https://github.com/astral-sh/setup-uv/commit/85aa0bf0c11c139baabd61df94702a838ff76d62"><code>85aa0bf</code></a>
chore: update known checksums for 0.5.10 (<a
href="https://redirect.github.com/astral-sh/setup-uv/issues/196">#196</a>)</li>
<li><a
href="https://github.com/astral-sh/setup-uv/commit/1f2cbfa7bb518f97c35c3cd1ac38271d6e45b512"><code>1f2cbfa</code></a>
Bump <code>@​types/node</code> from 22.10.1 to 22.10.2 (<a
href="https://redirect.github.com/astral-sh/setup-uv/issues/189">#189</a>)</li>
<li><a
href="https://github.com/astral-sh/setup-uv/commit/25b3ce6330ae41c3f95bd2addaf55f4db2d4820a"><code>25b3ce6</code></a>
chore: update known checksums for 0.5.9 (<a
href="https://redirect.github.com/astral-sh/setup-uv/issues/195">#195</a>)</li>
<li>Additional commits viewable in <a
href="https://github.com/astral-sh/setup-uv/compare/v4...v5">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=astral-sh/setup-uv&package-manager=github_actions&previous-version=4&new-version=5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/push.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
index 42245b14f..ddb2fb002 100644
--- a/.github/workflows/push.yml
+++ b/.github/workflows/push.yml
@@ -58,7 +58,7 @@ jobs:
           python-version: '3.9'
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@v5
 
       - name: Set go env
         run: |

From f8f804fe17ea49650068c521d8ac4cd8501ef22b Mon Sep 17 00:00:00 2001
From: Gleb Kanterov <kanterov@users.noreply.github.com>
Date: Mon, 13 Jan 2025 10:16:29 +0100
Subject: [PATCH 054/247] PythonMutator: update instrumentation (#2124)

## Changes
Update instrumentation for PythonMutator to handle `experimental/python`
config.

## Tests
Unit tests
---
 bundle/deploy/terraform/init.go      |  6 +++++-
 bundle/deploy/terraform/init_test.go | 22 +++++++++++++++++++++-
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/bundle/deploy/terraform/init.go b/bundle/deploy/terraform/init.go
index e69f0bf0f..d982354e1 100644
--- a/bundle/deploy/terraform/init.go
+++ b/bundle/deploy/terraform/init.go
@@ -232,7 +232,11 @@ func setUserAgentExtraEnvVar(environ map[string]string, b *bundle.Bundle) error
 	// Terraform provider to the CLI.
 	products := []string{"cli/" + build.GetInfo().Version}
 	if experimental := b.Config.Experimental; experimental != nil {
-		if experimental.PyDABs.Enabled {
+		hasPython := experimental.Python.Resources != nil || experimental.Python.Mutators != nil
+
+		if hasPython {
+			products = append(products, "databricks-pydabs/0.7.0")
+		} else if experimental.PyDABs.Enabled {
 			products = append(products, "databricks-pydabs/0.0.0")
 		}
 	}
diff --git a/bundle/deploy/terraform/init_test.go b/bundle/deploy/terraform/init_test.go
index 30ac9e301..c7a4ffe4a 100644
--- a/bundle/deploy/terraform/init_test.go
+++ b/bundle/deploy/terraform/init_test.go
@@ -248,7 +248,7 @@ func TestSetProxyEnvVars(t *testing.T) {
 	assert.ElementsMatch(t, []string{"HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY"}, maps.Keys(env))
 }
 
-func TestSetUserAgentExtraEnvVar(t *testing.T) {
+func TestSetUserAgentExtraEnvVar_PyDABs(t *testing.T) {
 	b := &bundle.Bundle{
 		BundleRootPath: t.TempDir(),
 		Config: config.Root{
@@ -268,6 +268,26 @@ func TestSetUserAgentExtraEnvVar(t *testing.T) {
 	}, env)
 }
 
+func TestSetUserAgentExtraEnvVar_Python(t *testing.T) {
+	b := &bundle.Bundle{
+		BundleRootPath: t.TempDir(),
+		Config: config.Root{
+			Experimental: &config.Experimental{
+				Python: config.Python{
+					Resources: []string{"my_project.resources:load_resources"},
+				},
+			},
+		},
+	}
+
+	env := make(map[string]string, 0)
+	err := setUserAgentExtraEnvVar(env, b)
+	require.NoError(t, err)
+	assert.Equal(t, map[string]string{
+		"DATABRICKS_USER_AGENT_EXTRA": "cli/0.0.0-dev databricks-pydabs/0.7.0",
+	}, env)
+}
+
 func TestInheritEnvVars(t *testing.T) {
 	t.Setenv("HOME", "/home/testuser")
 	t.Setenv("PATH", "/foo:/bar")

From 3e40a0c2f198e50d20aefb2ea39607147d416065 Mon Sep 17 00:00:00 2001
From: "Lennart Kats (databricks)" <lennart.kats@databricks.com>
Date: Mon, 13 Jan 2025 13:19:12 +0100
Subject: [PATCH 055/247] Encourage the use of root_path in production to
 ensure single deployment (#1712)

## Changes

This updates `mode: production` to allow `root_path` to indicate
uniqueness. Historically, we required `run_as` for this, which isn't
actually very effective for that purpose. `run_as` also had the problem
that it doesn't work for pipelines.

This is a cherry-pick from https://github.com/databricks/cli/pull/1387

---------

Co-authored-by: Pieter Noordhuis <pcnoordhuis@gmail.com>
---
 bundle/bundle.go                              |  3 +++
 bundle/config/mutator/process_target_mode.go  | 22 +++++++++++++++++--
 .../mutator/process_target_mode_test.go       | 21 ++++++++++++++++--
 bundle/config/mutator/select_target.go        |  7 ++++--
 bundle/config/root.go                         |  4 ++--
 libs/diag/diagnostic.go                       | 10 +++++++++
 6 files changed, 59 insertions(+), 8 deletions(-)

diff --git a/bundle/bundle.go b/bundle/bundle.go
index 1f5e2a294..3bf4ffb62 100644
--- a/bundle/bundle.go
+++ b/bundle/bundle.go
@@ -57,6 +57,9 @@ type Bundle struct {
 	// It is loaded from the bundle configuration files and mutators may update it.
 	Config config.Root
 
+	// Target stores a snapshot of the Root.Bundle.Target configuration when it was selected by SelectTarget.
+	Target *config.Target `json:"target_config,omitempty" bundle:"internal"`
+
 	// Metadata about the bundle deployment. This is the interface Databricks services
 	// rely on to integrate with bundles when they need additional information about
 	// a bundle deployment.
diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go
index 44b53681d..0fe6bd54f 100644
--- a/bundle/config/mutator/process_target_mode.go
+++ b/bundle/config/mutator/process_target_mode.go
@@ -2,6 +2,7 @@ package mutator
 
 import (
 	"context"
+	"fmt"
 	"strings"
 
 	"github.com/databricks/cli/bundle"
@@ -146,8 +147,21 @@ func validateProductionMode(ctx context.Context, b *bundle.Bundle, isPrincipalUs
 		}
 	}
 
-	if !isPrincipalUsed && !isRunAsSet(r) {
-		return diag.Errorf("'run_as' must be set for all jobs when using 'mode: production'")
+	// We need to verify that there is only a single deployment of the current target.
+	// The best way to enforce this is to explicitly set root_path.
+	advice := fmt.Sprintf(
+		"set 'workspace.root_path' to make sure only one copy is deployed. A common practice is to use a username or principal name in this path, i.e. root_path: /Workspace/Users/%s/.bundle/${bundle.name}/${bundle.target}",
+		b.Config.Workspace.CurrentUser.UserName,
+	)
+	if !isExplicitRootSet(b) {
+		if isRunAsSet(r) || isPrincipalUsed {
+			// Just setting run_as is not enough to guarantee a single deployment,
+			// and neither is setting a principal.
+			// We only show a warning for these cases since we didn't historically
+			// report an error for them.
+			return diag.Recommendationf("target with 'mode: production' should %s", advice)
+		}
+		return diag.Errorf("target with 'mode: production' must %s", advice)
 	}
 	return nil
 }
@@ -164,6 +178,10 @@ func isRunAsSet(r config.Resources) bool {
 	return true
 }
 
+func isExplicitRootSet(b *bundle.Bundle) bool {
+	return b.Target != nil && b.Target.Workspace != nil && b.Target.Workspace.RootPath != ""
+}
+
 func (m *processTargetMode) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	switch b.Config.Bundle.Mode {
 	case config.Development:
diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go
index 097c092a6..6df88d067 100644
--- a/bundle/config/mutator/process_target_mode_test.go
+++ b/bundle/config/mutator/process_target_mode_test.go
@@ -321,7 +321,7 @@ func TestProcessTargetModeProduction(t *testing.T) {
 	b := mockBundle(config.Production)
 
 	diags := validateProductionMode(context.Background(), b, false)
-	require.ErrorContains(t, diags.Error(), "run_as")
+	require.ErrorContains(t, diags.Error(), "target with 'mode: production' must set 'workspace.root_path' to make sure only one copy is deployed. A common practice is to use a username or principal name in this path, i.e. root_path: /Workspace/Users/lennart@company.com/.bundle/${bundle.name}/${bundle.target}")
 
 	b.Config.Workspace.StatePath = "/Shared/.bundle/x/y/state"
 	b.Config.Workspace.ArtifactPath = "/Shared/.bundle/x/y/artifacts"
@@ -329,7 +329,7 @@ func TestProcessTargetModeProduction(t *testing.T) {
 	b.Config.Workspace.ResourcePath = "/Shared/.bundle/x/y/resources"
 
 	diags = validateProductionMode(context.Background(), b, false)
-	require.ErrorContains(t, diags.Error(), "production")
+	require.ErrorContains(t, diags.Error(), "target with 'mode: production' must set 'workspace.root_path' to make sure only one copy is deployed. A common practice is to use a username or principal name in this path, i.e. root_path: /Workspace/Users/lennart@company.com/.bundle/${bundle.name}/${bundle.target}")
 
 	permissions := []resources.Permission{
 		{
@@ -375,6 +375,23 @@ func TestProcessTargetModeProductionOkForPrincipal(t *testing.T) {
 	require.NoError(t, diags.Error())
 }
 
+func TestProcessTargetModeProductionOkWithRootPath(t *testing.T) {
+	b := mockBundle(config.Production)
+
+	// Our target has all kinds of problems when not using service principals ...
+	diags := validateProductionMode(context.Background(), b, false)
+	require.Error(t, diags.Error())
+
+	// ... but we're okay if we specify a root path
+	b.Target = &config.Target{
+		Workspace: &config.Workspace{
+			RootPath: "some-root-path",
+		},
+	}
+	diags = validateProductionMode(context.Background(), b, false)
+	require.NoError(t, diags.Error())
+}
+
 // Make sure that we have test coverage for all resource types
 func TestAllResourcesMocked(t *testing.T) {
 	b := mockBundle(config.Development)
diff --git a/bundle/config/mutator/select_target.go b/bundle/config/mutator/select_target.go
index 178686b6e..ce18da4f5 100644
--- a/bundle/config/mutator/select_target.go
+++ b/bundle/config/mutator/select_target.go
@@ -15,6 +15,7 @@ type selectTarget struct {
 }
 
 // SelectTarget merges the specified target into the root configuration.
+// After merging, it removes the 'Targets' section from the configuration.
 func SelectTarget(name string) bundle.Mutator {
 	return &selectTarget{
 		name: name,
@@ -31,7 +32,7 @@ func (m *selectTarget) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnosti
 	}
 
 	// Get specified target
-	_, ok := b.Config.Targets[m.name]
+	target, ok := b.Config.Targets[m.name]
 	if !ok {
 		return diag.Errorf("%s: no such target. Available targets: %s", m.name, strings.Join(maps.Keys(b.Config.Targets), ", "))
 	}
@@ -43,13 +44,15 @@ func (m *selectTarget) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnosti
 	}
 
 	// Store specified target in configuration for reference.
+	b.Target = target
 	b.Config.Bundle.Target = m.name
 
 	// We do this for backward compatibility.
 	// TODO: remove when Environments section is not supported anymore.
 	b.Config.Bundle.Environment = b.Config.Bundle.Target
 
-	// Clear targets after loading.
+	// Cleanup the original targets and environments sections since they
+	// show up in the JSON output of the 'summary' and 'validate' commands.
 	b.Config.Targets = nil
 	b.Config.Environments = nil
 
diff --git a/bundle/config/root.go b/bundle/config/root.go
index 91c15fd9d..21804110a 100644
--- a/bundle/config/root.go
+++ b/bundle/config/root.go
@@ -47,8 +47,8 @@ type Root struct {
 
 	// Targets can be used to differentiate settings and resources between
 	// bundle deployment targets (e.g. development, staging, production).
-	// If not specified, the code below initializes this field with a
-	// single default-initialized target called "default".
+	// Note that this field is set to 'nil' by the SelectTarget mutator;
+	// use bundle.Bundle.Target to access the selected target configuration.
 	Targets map[string]*Target `json:"targets,omitempty"`
 
 	// DEPRECATED. Left for backward compatibility with Targets
diff --git a/libs/diag/diagnostic.go b/libs/diag/diagnostic.go
index a4f8c7b6b..0c7699b4e 100644
--- a/libs/diag/diagnostic.go
+++ b/libs/diag/diagnostic.go
@@ -86,6 +86,16 @@ func Infof(format string, args ...any) Diagnostics {
 	}
 }
 
+// Recommendationf creates a new recommendation diagnostic.
+func Recommendationf(format string, args ...any) Diagnostics {
+	return []Diagnostic{
+		{
+			Severity: Recommendation,
+			Summary:  fmt.Sprintf(format, args...),
+		},
+	}
+}
+
 // Diagnostics holds zero or more instances of [Diagnostic].
 type Diagnostics []Diagnostic
 

From cae21b36de7451f7a78ede1b44711ebdd55cd7f4 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 13 Jan 2025 13:31:09 +0100
Subject: [PATCH 056/247] Add a test re using variable in host (#2117)

Related issue: https://github.com/databricks/cli/issues/2095
---
 .../bundle/variables/host/databricks.yml      | 10 +++++
 acceptance/bundle/variables/host/output.txt   | 38 +++++++++++++++++++
 acceptance/bundle/variables/host/script       |  2 +
 3 files changed, 50 insertions(+)
 create mode 100644 acceptance/bundle/variables/host/databricks.yml
 create mode 100644 acceptance/bundle/variables/host/output.txt
 create mode 100644 acceptance/bundle/variables/host/script

diff --git a/acceptance/bundle/variables/host/databricks.yml b/acceptance/bundle/variables/host/databricks.yml
new file mode 100644
index 000000000..b25020a1f
--- /dev/null
+++ b/acceptance/bundle/variables/host/databricks.yml
@@ -0,0 +1,10 @@
+bundle:
+  name: host
+
+variables:
+  host:
+    default: https://nonexistent123.staging.cloud.databricks.com
+
+workspace:
+  # This is currently not supported
+  host: ${var.host}
diff --git a/acceptance/bundle/variables/host/output.txt b/acceptance/bundle/variables/host/output.txt
new file mode 100644
index 000000000..89342908c
--- /dev/null
+++ b/acceptance/bundle/variables/host/output.txt
@@ -0,0 +1,38 @@
+
+>>> errcode $CLI bundle validate -o json
+Error: failed during request visitor: parse "https://${var.host}": invalid character "{" in host name
+
+{
+  "bundle": {
+    "environment": "default",
+    "name": "host",
+    "target": "default"
+  },
+  "sync": {
+    "paths": [
+      "."
+    ]
+  },
+  "targets": null,
+  "variables": {
+    "host": {
+      "default": "https://nonexistent123.staging.cloud.databricks.com"
+    }
+  },
+  "workspace": {
+    "host": "${var.host}"
+  }
+}
+Exit code: 1
+
+>>> errcode $CLI bundle validate
+Error: failed during request visitor: parse "https://${var.host}": invalid character "{" in host name
+
+Name: host
+Target: default
+Workspace:
+  Host: ${var.host}
+
+Found 1 error
+
+Exit code: 1
diff --git a/acceptance/bundle/variables/host/script b/acceptance/bundle/variables/host/script
new file mode 100644
index 000000000..90e083627
--- /dev/null
+++ b/acceptance/bundle/variables/host/script
@@ -0,0 +1,2 @@
+trace errcode $CLI bundle validate -o json
+trace errcode $CLI bundle validate

From 1ead1b2e361c6918f8e43f1d4a8b00f931b7426e Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 13 Jan 2025 14:01:31 +0100
Subject: [PATCH 057/247] Move merge fix-ups after variable resolution (#2125)

## Changes
Move mutator.Merge{JobClusters,JobParameters,JobTasks,PipelineClusters}
after variable resolution. This helps with the case when key contains a
variable.

@pietern mentioned here
https://github.com/databricks/cli/pull/2101#pullrequestreview-2539168762
it should be safe.

## Tests
Existing acceptance that was capturing the bug is updated with corrected
output.
---
 .../override/job_cluster_var/databricks.yml    |  1 -
 .../bundle/override/job_cluster_var/output.txt | 18 ++++--------------
 bundle/phases/initialize.go                    | 10 ++++++----
 3 files changed, 10 insertions(+), 19 deletions(-)

diff --git a/acceptance/bundle/override/job_cluster_var/databricks.yml b/acceptance/bundle/override/job_cluster_var/databricks.yml
index 546cc2d8a..48e68c926 100644
--- a/acceptance/bundle/override/job_cluster_var/databricks.yml
+++ b/acceptance/bundle/override/job_cluster_var/databricks.yml
@@ -20,7 +20,6 @@ targets:
       jobs:
         foo:
           job_clusters:
-            # This does not work because merging is done before resolution
             - job_cluster_key: "${var.mykey}"
               new_cluster:
                 node_type_id: i3.xlarge
diff --git a/acceptance/bundle/override/job_cluster_var/output.txt b/acceptance/bundle/override/job_cluster_var/output.txt
index dee2a3b5b..cb76de5a8 100644
--- a/acceptance/bundle/override/job_cluster_var/output.txt
+++ b/acceptance/bundle/override/job_cluster_var/output.txt
@@ -9,17 +9,12 @@
     "edit_mode": "UI_LOCKED",
     "format": "MULTI_TASK",
     "job_clusters": [
-      {
-        "job_cluster_key": "key",
-        "new_cluster": {
-          "spark_version": "13.3.x-scala2.12"
-        }
-      },
       {
         "job_cluster_key": "key",
         "new_cluster": {
           "node_type_id": "i3.xlarge",
-          "num_workers": 1
+          "num_workers": 1,
+          "spark_version": "13.3.x-scala2.12"
         }
       }
     ],
@@ -51,17 +46,12 @@ Validation OK!
     "edit_mode": "UI_LOCKED",
     "format": "MULTI_TASK",
     "job_clusters": [
-      {
-        "job_cluster_key": "key",
-        "new_cluster": {
-          "spark_version": "13.3.x-scala2.12"
-        }
-      },
       {
         "job_cluster_key": "key",
         "new_cluster": {
           "node_type_id": "i3.2xlarge",
-          "num_workers": 4
+          "num_workers": 4,
+          "spark_version": "13.3.x-scala2.12"
         }
       }
     ],
diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go
index f7b3cd608..913685bcf 100644
--- a/bundle/phases/initialize.go
+++ b/bundle/phases/initialize.go
@@ -33,10 +33,6 @@ func Initialize() bundle.Mutator {
 			// If it is an ancestor, this updates all paths to be relative to the sync root path.
 			mutator.SyncInferRoot(),
 
-			mutator.MergeJobClusters(),
-			mutator.MergeJobParameters(),
-			mutator.MergeJobTasks(),
-			mutator.MergePipelineClusters(),
 			mutator.InitializeWorkspaceClient(),
 			mutator.PopulateCurrentUser(),
 			mutator.LoadGitDetails(),
@@ -70,6 +66,12 @@ func Initialize() bundle.Mutator {
 				"workspace",
 				"variables",
 			),
+
+			mutator.MergeJobClusters(),
+			mutator.MergeJobParameters(),
+			mutator.MergeJobTasks(),
+			mutator.MergePipelineClusters(),
+
 			// Provide permission config errors & warnings after initializing all variables
 			permissions.PermissionDiagnostics(),
 			mutator.SetRunAs(),

From 244a5b6bc65ad336b1052bc2d23c81fe9483cfba Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Jan 2025 13:26:35 +0000
Subject: [PATCH 058/247] Bump golang.org/x/oauth2 from 0.24.0 to 0.25.0
 (#2080)

Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from
0.24.0 to 0.25.0.
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/golang/oauth2/commit/49a531d12a9ad6fa9f5a070d577ac752ada772c9"><code>49a531d</code></a>
all: make method and struct comments match the names</li>
<li>See full diff in <a
href="https://github.com/golang/oauth2/compare/v0.24.0...v0.25.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/oauth2&package-manager=go_modules&previous-version=0.24.0&new-version=0.25.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 86bc1c368..4f8b57d0a 100644
--- a/go.mod
+++ b/go.mod
@@ -26,7 +26,7 @@ require (
 	github.com/wI2L/jsondiff v0.6.1 // MIT
 	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
 	golang.org/x/mod v0.22.0
-	golang.org/x/oauth2 v0.24.0
+	golang.org/x/oauth2 v0.25.0
 	golang.org/x/sync v0.10.0
 	golang.org/x/term v0.27.0
 	golang.org/x/text v0.21.0
diff --git a/go.sum b/go.sum
index f6cf79607..84587c850 100644
--- a/go.sum
+++ b/go.sum
@@ -207,8 +207,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
-golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
+golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70=
+golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

From f8ab384bfba3753b71d3583f649847734b8af4b6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Jan 2025 13:26:47 +0000
Subject: [PATCH 059/247] Bump github.com/hashicorp/hc-install from 0.9.0 to
 0.9.1 (#2079)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bumps
[github.com/hashicorp/hc-install](https://github.com/hashicorp/hc-install)
from 0.9.0 to 0.9.1.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/hashicorp/hc-install/releases">github.com/hashicorp/hc-install's
releases</a>.</em></p>
<blockquote>
<h2>v0.9.1</h2>
<h2>What's Changed</h2>
<ul>
<li>build(deps): bump github.com/go-git/go-git/v5 from 5.12.0 to 5.13.0
by <a href="https://github.com/dependabot"><code>@​dependabot</code></a>
in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/268">hashicorp/hc-install#268</a></li>
<li>build(deps): bump github.com/ProtonMail/go-crypto from 1.1.0 to
1.1.2 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/261">hashicorp/hc-install#261</a></li>
<li>build(deps): bump github.com/ProtonMail/go-crypto from 1.1.0-alpha.2
to 1.1.0 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/259">hashicorp/hc-install#259</a></li>
<li>build(deps): bump github.com/ProtonMail/go-crypto from 1.1.2 to
1.1.3 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/263">hashicorp/hc-install#263</a></li>
<li>build(deps): bump golang.org/x/mod from 0.21.0 to 0.22.0 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/262">hashicorp/hc-install#262</a></li>
</ul>
<h2>New Contributors</h2>
<ul>
<li><a
href="https://github.com/imakewebthings"><code>@​imakewebthings</code></a>
made their first contribution in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/252">hashicorp/hc-install#252</a></li>
</ul>
<p><strong>Full Changelog</strong>: <a
href="https://github.com/hashicorp/hc-install/compare/v0.9.0...v0.9.1">https://github.com/hashicorp/hc-install/compare/v0.9.0...v0.9.1</a></p>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/hashicorp/hc-install/commit/a9cdf85469c551496a66528c239a4f30dbeb8cc6"><code>a9cdf85</code></a>
Prepare for 0.9.1 release (<a
href="https://redirect.github.com/hashicorp/hc-install/issues/269">#269</a>)</li>
<li><a
href="https://github.com/hashicorp/hc-install/commit/18d08ba3e4137df00d1ba720a41173b59c478535"><code>18d08ba</code></a>
build(deps): Bump workflows to latest trusted versions (<a
href="https://redirect.github.com/hashicorp/hc-install/issues/266">#266</a>)</li>
<li><a
href="https://github.com/hashicorp/hc-install/commit/e716f0ac3e1eb3254f6b1f99c107858d5c25e8b9"><code>e716f0a</code></a>
build(deps): bump github.com/go-git/go-git/v5 from 5.12.0 to 5.13.0 (<a
href="https://redirect.github.com/hashicorp/hc-install/issues/268">#268</a>)</li>
<li><a
href="https://github.com/hashicorp/hc-install/commit/cca0f6dd33872b605174f3cf9bd3c3cbafc70138"><code>cca0f6d</code></a>
ci: Report code coverage (<a
href="https://redirect.github.com/hashicorp/hc-install/issues/264">#264</a>)</li>
<li><a
href="https://github.com/hashicorp/hc-install/commit/131f8ffdb00c2ddc0e9e56d05d6f80f143f443a1"><code>131f8ff</code></a>
build(deps): bump github.com/ProtonMail/go-crypto from 1.1.2 to 1.1.3
(<a
href="https://redirect.github.com/hashicorp/hc-install/issues/263">#263</a>)</li>
<li><a
href="https://github.com/hashicorp/hc-install/commit/2609a7830ad02abe7b3d529bfd810dd1075a706b"><code>2609a78</code></a>
build(deps): bump golang.org/x/mod from 0.21.0 to 0.22.0 (<a
href="https://redirect.github.com/hashicorp/hc-install/issues/262">#262</a>)</li>
<li><a
href="https://github.com/hashicorp/hc-install/commit/b9043f8dd14da5ff987dcc52016868bcabb6d31f"><code>b9043f8</code></a>
build(deps): bump github.com/ProtonMail/go-crypto from 1.1.0 to 1.1.2
(<a
href="https://redirect.github.com/hashicorp/hc-install/issues/261">#261</a>)</li>
<li><a
href="https://github.com/hashicorp/hc-install/commit/c1dc8ac751329f4c61a0f3ab33ac8f92734d5615"><code>c1dc8ac</code></a>
build(deps): bump github.com/ProtonMail/go-crypto from 1.1.0-alpha.2 to
1.1.0...</li>
<li><a
href="https://github.com/hashicorp/hc-install/commit/8ed2e0f78e5a0cfec6b74e6d8245d0201c77e05a"><code>8ed2e0f</code></a>
build(deps): Bump workflows to latest trusted versions (<a
href="https://redirect.github.com/hashicorp/hc-install/issues/258">#258</a>)</li>
<li><a
href="https://github.com/hashicorp/hc-install/commit/7a0461e71355166b7b4a5c3a168c8ceca3171006"><code>7a0461e</code></a>
build(deps): Bump workflows to latest trusted versions (<a
href="https://redirect.github.com/hashicorp/hc-install/issues/257">#257</a>)</li>
<li>Additional commits viewable in <a
href="https://github.com/hashicorp/hc-install/compare/v0.9.0...v0.9.1">compare
view</a></li>
</ul>
</details>
<br />

<details>
<summary>Most Recent Ignore Conditions Applied to This Pull
Request</summary>

| Dependency Name | Ignore Conditions |
| --- | --- |
| github.com/hashicorp/hc-install | [>= 0.8.a, < 0.9] |
</details>


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/hashicorp/hc-install&package-manager=go_modules&previous-version=0.9.0&new-version=0.9.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod |  4 ++--
 go.sum | 24 ++++++++++++------------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/go.mod b/go.mod
index 4f8b57d0a..867fbdf3c 100644
--- a/go.mod
+++ b/go.mod
@@ -11,7 +11,7 @@ require (
 	github.com/fatih/color v1.18.0 // MIT
 	github.com/google/uuid v1.6.0 // BSD-3-Clause
 	github.com/hashicorp/go-version v1.7.0 // MPL 2.0
-	github.com/hashicorp/hc-install v0.9.0 // MPL 2.0
+	github.com/hashicorp/hc-install v0.9.1 // MPL 2.0
 	github.com/hashicorp/terraform-exec v0.21.0 // MPL 2.0
 	github.com/hashicorp/terraform-json v0.23.0 // MPL 2.0
 	github.com/hexops/gotextdiff v1.0.3 // BSD 3-Clause "New" or "Revised" License
@@ -38,7 +38,7 @@ require (
 	cloud.google.com/go/auth v0.4.2 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
 	cloud.google.com/go/compute/metadata v0.3.0 // indirect
-	github.com/ProtonMail/go-crypto v1.1.0-alpha.2 // indirect
+	github.com/ProtonMail/go-crypto v1.1.3 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
diff --git a/go.sum b/go.sum
index 84587c850..0e9d13ae2 100644
--- a/go.sum
+++ b/go.sum
@@ -12,8 +12,8 @@ github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7r
 github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
-github.com/ProtonMail/go-crypto v1.1.0-alpha.2 h1:bkyFVUP+ROOARdgCiJzNQo2V2kiB97LyUpzH9P6Hrlg=
-github.com/ProtonMail/go-crypto v1.1.0-alpha.2/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE=
+github.com/ProtonMail/go-crypto v1.1.3 h1:nRBOetoydLeUb4nHajyO2bKqMLfWQ/ZPwkXqXxPxCFk=
+github.com/ProtonMail/go-crypto v1.1.3/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE=
 github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
 github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
 github.com/briandowns/spinner v1.23.1 h1:t5fDPmScwUjozhDj4FA46p5acZWIPXYE30qW2Ptu650=
@@ -30,8 +30,8 @@ github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vc
 github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
-github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
+github.com/cyphar/filepath-securejoin v0.2.5 h1:6iR5tXJ/e6tJZzzdMc1km3Sa7RRIVBKAK32O2s7AYfo=
+github.com/cyphar/filepath-securejoin v0.2.5/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/databricks/databricks-sdk-go v0.54.0 h1:L8gsA3NXs+uYU3QtW/OUgjxMQxOH24k0MT9JhB3zLlM=
 github.com/databricks/databricks-sdk-go v0.54.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -50,10 +50,10 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
-github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
-github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
-github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys=
-github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY=
+github.com/go-git/go-billy/v5 v5.6.0 h1:w2hPNtoehvJIxR00Vb4xX94qHQi/ApZfX+nBE2Cjio8=
+github.com/go-git/go-billy/v5 v5.6.0/go.mod h1:sFDq7xD3fn3E0GOwUSZqHo9lrkmx8xJhA0ZrfvjBRGM=
+github.com/go-git/go-git/v5 v5.13.0 h1:vLn5wlGIh/X78El6r3Jr+30W16Blk0CTcxTYcYPWi5E=
+github.com/go-git/go-git/v5 v5.13.0/go.mod h1:Wjo7/JyVKtQgUNdXYXIepzWfJQkUEIGvkvVkiXRR/zw=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
 github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -103,8 +103,8 @@ github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISH
 github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
 github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
 github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/hashicorp/hc-install v0.9.0 h1:2dIk8LcvANwtv3QZLckxcjyF5w8KVtiMxu6G6eLhghE=
-github.com/hashicorp/hc-install v0.9.0/go.mod h1:+6vOP+mf3tuGgMApVYtmsnDoKWMDcFXeTxCACYZ8SFg=
+github.com/hashicorp/hc-install v0.9.1 h1:gkqTfE3vVbafGQo6VZXcy2v5yoz2bE0+nhZXruCuODQ=
+github.com/hashicorp/hc-install v0.9.1/go.mod h1:pWWvN/IrfeBK4XPeXXYkL6EjMufHkCK5DvwxeLKuBf0=
 github.com/hashicorp/terraform-exec v0.21.0 h1:uNkLAe95ey5Uux6KJdua6+cv8asgILFVWkd/RG0D2XQ=
 github.com/hashicorp/terraform-exec v0.21.0/go.mod h1:1PPeMYou+KDUSSeRE9szMZ/oHf4fYUmB923Wzbq1ICg=
 github.com/hashicorp/terraform-json v0.23.0 h1:sniCkExU4iKtTADReHzACkk8fnpQXrdD2xoR+lppBkI=
@@ -141,8 +141,8 @@ github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDj
 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
 github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
 github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
-github.com/skeema/knownhosts v1.2.2 h1:Iug2P4fLmDw9f41PB6thxUkNUkJzB5i+1/exaj40L3A=
-github.com/skeema/knownhosts v1.2.2/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo=
+github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0LY=
+github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M=
 github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
 github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=

From 8234604cad5db7263684b225375bdfaf25c3acf0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Jan 2025 13:26:55 +0000
Subject: [PATCH 060/247] Bump golang.org/x/term from 0.27.0 to 0.28.0 (#2078)

Bumps [golang.org/x/term](https://github.com/golang/term) from 0.27.0 to
0.28.0.
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/golang/term/commit/40b02d69cd8f2efc8aeb262071f74fb4319b6661"><code>40b02d6</code></a>
go.mod: update golang.org/x dependencies</li>
<li>See full diff in <a
href="https://github.com/golang/term/compare/v0.27.0...v0.28.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/term&package-manager=go_modules&previous-version=0.27.0&new-version=0.28.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 4 ++--
 go.sum | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/go.mod b/go.mod
index 867fbdf3c..ed2ff12ad 100644
--- a/go.mod
+++ b/go.mod
@@ -28,7 +28,7 @@ require (
 	golang.org/x/mod v0.22.0
 	golang.org/x/oauth2 v0.25.0
 	golang.org/x/sync v0.10.0
-	golang.org/x/term v0.27.0
+	golang.org/x/term v0.28.0
 	golang.org/x/text v0.21.0
 	gopkg.in/ini.v1 v1.67.0 // Apache 2.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -69,7 +69,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	golang.org/x/crypto v0.31.0 // indirect
 	golang.org/x/net v0.33.0 // indirect
-	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/sys v0.29.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	google.golang.org/api v0.182.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect
diff --git a/go.sum b/go.sum
index 0e9d13ae2..2b9290b71 100644
--- a/go.sum
+++ b/go.sum
@@ -224,10 +224,10 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
-golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
+golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
+golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
+golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=

From a6412e43345f3ee3f048a04c7bef4a9d2c4372ba Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 13 Jan 2025 17:12:03 +0100
Subject: [PATCH 061/247] Remove redundant lines from PrepareReplacementsUser
 (#2130)

They are not necessary because they are added below. Also, they will
cause a crash if u.Name is nil.
---
 libs/testdiff/golden.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/libs/testdiff/golden.go b/libs/testdiff/golden.go
index 02213c88a..08d1e9608 100644
--- a/libs/testdiff/golden.go
+++ b/libs/testdiff/golden.go
@@ -185,8 +185,6 @@ func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam.
 		u.DisplayName,
 		u.UserName,
 		iamutil.GetShortUserName(&u),
-		u.Name.FamilyName,
-		u.Name.GivenName,
 	}
 	if u.Name != nil {
 		names = append(names, u.Name.FamilyName)

From 913e10a0375e85dbe2773e47ee933f4103a8f2f0 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Mon, 13 Jan 2025 17:43:48 +0100
Subject: [PATCH 062/247] Added support for Databricks Apps in DABs (#1928)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Changes
Now it's possible to configure new `app` resource in bundle and point it
to the custom `source_code_path` location where Databricks App code is
defined.

On `databricks bundle deploy` DABs will create an app. All consecutive
`databricks bundle deploy` execution will update an existing app if
there are any updated

On `databricks bundle run <my_app>` DABs will execute app deployment. If
the app is not started yet, it will start the app first.

### Bundle configuration

```
bundle:
  name: apps

variables:
  my_job_id:
    description: "ID of job to run app"
    lookup:
      job: "My Job"
  databricks_name:
    description: "Name for app user"
  additional_flags:
    description: "Additional flags to run command app"
    default: ""
  my_app_config:
    type: complex
    description: "Configuration for my Databricks App"
    default:
      command:
        - flask
        - --app
        - hello
        - run
        - ${var.additional_flags}
      env:
        - name: DATABRICKS_NAME
          value: ${var.databricks_name}

resources:
  apps:
    my_app:
      name: "anester-app" # required and has to be unique
      description: "My App"
      source_code_path: ./app # required and points to location of app code
      config: ${var.my_app_config}
      resources:
        - name: "my-job"
          description: "A job for app to be able to run"
          job:
            id: ${var.my_job_id}
            permission: "CAN_MANAGE_RUN"
      permissions:
        - user_name: "foo@bar.com"
          level: "CAN_VIEW"
        - service_principal_name: "my_sp"
          level: "CAN_MANAGE"

targets:
  dev:
    variables:
      databricks_name: "Andrew (from dev)"
      additional_flags: --debug

  prod:
    variables:
      databricks_name: "Andrew (from prod)"
```

### Execution
1. `databricks bundle deploy -t dev`
2. `databricks bundle run my_app -t dev`

**If app is started**
```
✓ Getting the status of the app my-app
✓ App is in RUNNING state
✓ Preparing source code for new app deployment.
✓ Deployment is pending
✓ Starting app with command: flask --app hello run --debug
✓ App started successfully
You can access the app at <app-url>
```

**If app is not started**
```
✓ Getting the status of the app my-app
✓ App is in UNAVAILABLE state
✓ Starting the app my-app
✓ App is starting...
....
✓ App is starting...
✓ App is started!
✓ Preparing source code for new app deployment.
✓ Downloading source code from /Workspace/Users/...
✓ Starting app with command: flask --app hello run --debug
✓ App started successfully
You can access the app at <app-url>
```

## Tests
Added unit and config tests + manual test.

```
--- PASS: TestAccDeployBundleWithApp (404.59s)
PASS
coverage: 36.8% of statements in ./...
ok      github.com/databricks/cli/internal/bundle       405.035s        coverage: 36.8% of statements in ./...
```
---
 bundle/apps/interpolate_variables.go          |  50 +++
 bundle/apps/interpolate_variables_test.go     |  49 +++
 bundle/apps/upload_config.go                  |  97 +++++
 bundle/apps/upload_config_test.go             |  75 ++++
 bundle/apps/validate.go                       |  53 +++
 bundle/apps/validate_test.go                  |  97 +++++
 bundle/config/generate/app.go                 |  37 ++
 bundle/config/mutator/apply_presets.go        |   2 +
 .../apply_source_linked_deployment_preset.go  |  16 +
 ...ly_source_linked_deployment_preset_test.go |  21 +-
 bundle/config/mutator/merge_apps.go           |  45 ++
 bundle/config/mutator/merge_apps_test.go      |  73 ++++
 .../mutator/process_target_mode_test.go       |  15 +
 bundle/config/mutator/run_as.go               |  10 +
 bundle/config/mutator/run_as_test.go          | 132 ++++--
 bundle/config/mutator/translate_paths.go      |   1 +
 bundle/config/mutator/translate_paths_apps.go |  28 ++
 .../mutator/translate_paths_apps_test.go      |  57 +++
 bundle/config/resources.go                    | 117 ++++--
 bundle/config/resources/apps.go               |  70 ++++
 bundle/deploy/terraform/convert.go            |  20 +
 bundle/deploy/terraform/convert_test.go       |  57 +++
 bundle/deploy/terraform/interpolate.go        |   2 +
 bundle/deploy/terraform/interpolate_test.go   |   2 +
 bundle/deploy/terraform/tfdyn/convert_app.go  |  55 +++
 .../terraform/tfdyn/convert_app_test.go       | 156 +++++++
 bundle/deploy/terraform/util.go               |   7 +-
 bundle/deploy/terraform/util_test.go          |   2 +-
 bundle/internal/schema/annotations.yml        | 161 +++++++
 bundle/permissions/mutator.go                 |   4 +
 bundle/permissions/mutator_test.go            |   8 +
 bundle/phases/deploy.go                       |   3 +
 bundle/phases/initialize.go                   |   4 +
 bundle/run/app.go                             | 212 ++++++++++
 bundle/run/app_test.go                        | 216 ++++++++++
 bundle/run/runner.go                          |   8 +-
 bundle/schema/jsonschema.json                 | 394 ++++++++++++++++++
 bundle/tests/apps/databricks.yml              |  71 ++++
 bundle/tests/apps_test.go                     |  60 +++
 bundle/tests/loader.go                        |   1 +
 cmd/bundle/generate.go                        |   1 +
 cmd/bundle/generate/app.go                    | 166 ++++++++
 cmd/bundle/generate/utils.go                  |  32 ++
 integration/bundle/apps_test.go               | 113 +++++
 .../apps/databricks_template_schema.json      |  24 ++
 .../bundle/bundles/apps/template/app/app.py   |  15 +
 .../bundles/apps/template/databricks.yml.tmpl |  42 ++
 .../bundles/apps/template/hello_world.py      |   1 +
 integration/bundle/helpers_test.go            |  11 +
 libs/dyn/merge/elements_by_key.go             |  28 +-
 libs/dyn/merge/elements_by_key_test.go        |  39 ++
 51 files changed, 2870 insertions(+), 90 deletions(-)
 create mode 100644 bundle/apps/interpolate_variables.go
 create mode 100644 bundle/apps/interpolate_variables_test.go
 create mode 100644 bundle/apps/upload_config.go
 create mode 100644 bundle/apps/upload_config_test.go
 create mode 100644 bundle/apps/validate.go
 create mode 100644 bundle/apps/validate_test.go
 create mode 100644 bundle/config/generate/app.go
 create mode 100644 bundle/config/mutator/merge_apps.go
 create mode 100644 bundle/config/mutator/merge_apps_test.go
 create mode 100644 bundle/config/mutator/translate_paths_apps.go
 create mode 100644 bundle/config/mutator/translate_paths_apps_test.go
 create mode 100644 bundle/config/resources/apps.go
 create mode 100644 bundle/deploy/terraform/tfdyn/convert_app.go
 create mode 100644 bundle/deploy/terraform/tfdyn/convert_app_test.go
 create mode 100644 bundle/run/app.go
 create mode 100644 bundle/run/app_test.go
 create mode 100644 bundle/tests/apps/databricks.yml
 create mode 100644 bundle/tests/apps_test.go
 create mode 100644 cmd/bundle/generate/app.go
 create mode 100644 integration/bundle/apps_test.go
 create mode 100644 integration/bundle/bundles/apps/databricks_template_schema.json
 create mode 100644 integration/bundle/bundles/apps/template/app/app.py
 create mode 100644 integration/bundle/bundles/apps/template/databricks.yml.tmpl
 create mode 100644 integration/bundle/bundles/apps/template/hello_world.py

diff --git a/bundle/apps/interpolate_variables.go b/bundle/apps/interpolate_variables.go
new file mode 100644
index 000000000..f88e7e9db
--- /dev/null
+++ b/bundle/apps/interpolate_variables.go
@@ -0,0 +1,50 @@
+package apps
+
+import (
+	"context"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/dynvar"
+)
+
+type interpolateVariables struct{}
+
+func (i *interpolateVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	pattern := dyn.NewPattern(
+		dyn.Key("resources"),
+		dyn.Key("apps"),
+		dyn.AnyKey(),
+		dyn.Key("config"),
+	)
+
+	tfToConfigMap := map[string]string{}
+	for k, r := range config.SupportedResources() {
+		tfToConfigMap[r.TerraformResourceName] = k
+	}
+
+	err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
+		return dyn.MapByPattern(root, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+			return dynvar.Resolve(v, func(path dyn.Path) (dyn.Value, error) {
+				key, ok := tfToConfigMap[path[0].Key()]
+				if ok {
+					path = dyn.NewPath(dyn.Key("resources"), dyn.Key(key)).Append(path[1:]...)
+				}
+
+				return dyn.GetByPath(root, path)
+			})
+		})
+	})
+
+	return diag.FromErr(err)
+}
+
+func (i *interpolateVariables) Name() string {
+	return "apps.InterpolateVariables"
+}
+
+func InterpolateVariables() bundle.Mutator {
+	return &interpolateVariables{}
+}
diff --git a/bundle/apps/interpolate_variables_test.go b/bundle/apps/interpolate_variables_test.go
new file mode 100644
index 000000000..a2909006f
--- /dev/null
+++ b/bundle/apps/interpolate_variables_test.go
@@ -0,0 +1,49 @@
+package apps
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/databricks-sdk-go/service/apps"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAppInterpolateVariables(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Apps: map[string]*resources.App{
+					"my_app_1": {
+						App: &apps.App{
+							Name: "my_app_1",
+						},
+						Config: map[string]any{
+							"command": []string{"echo", "hello"},
+							"env": []map[string]string{
+								{"name": "JOB_ID", "value": "${databricks_job.my_job.id}"},
+							},
+						},
+					},
+					"my_app_2": {
+						App: &apps.App{
+							Name: "my_app_2",
+						},
+					},
+				},
+				Jobs: map[string]*resources.Job{
+					"my_job": {
+						ID: "123",
+					},
+				},
+			},
+		},
+	}
+
+	diags := bundle.Apply(context.Background(), b, InterpolateVariables())
+	require.Empty(t, diags)
+	require.Equal(t, []any([]any{map[string]any{"name": "JOB_ID", "value": "123"}}), b.Config.Resources.Apps["my_app_1"].Config["env"])
+	require.Nil(t, b.Config.Resources.Apps["my_app_2"].Config)
+}
diff --git a/bundle/apps/upload_config.go b/bundle/apps/upload_config.go
new file mode 100644
index 000000000..5c58c5c6f
--- /dev/null
+++ b/bundle/apps/upload_config.go
@@ -0,0 +1,97 @@
+package apps
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"path"
+	"strings"
+	"sync"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/deploy"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/filer"
+	"golang.org/x/sync/errgroup"
+
+	"gopkg.in/yaml.v3"
+)
+
+type uploadConfig struct {
+	filerFactory deploy.FilerFactory
+}
+
+func (u *uploadConfig) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	var diags diag.Diagnostics
+	errGroup, ctx := errgroup.WithContext(ctx)
+
+	mu := sync.Mutex{}
+	for key, app := range b.Config.Resources.Apps {
+		// If the app has a config, we need to deploy it first.
+		// It means we need to write app.yml file with the content of the config field
+		// to the remote source code path of the app.
+		if app.Config != nil {
+			appPath := strings.TrimPrefix(app.SourceCodePath, b.Config.Workspace.FilePath)
+
+			buf, err := configToYaml(app)
+			if err != nil {
+				return diag.FromErr(err)
+			}
+
+			f, err := u.filerFactory(b)
+			if err != nil {
+				return diag.FromErr(err)
+			}
+
+			errGroup.Go(func() error {
+				err := f.Write(ctx, path.Join(appPath, "app.yml"), buf, filer.OverwriteIfExists)
+				if err != nil {
+					mu.Lock()
+					diags = append(diags, diag.Diagnostic{
+						Severity:  diag.Error,
+						Summary:   "Failed to save config",
+						Detail:    fmt.Sprintf("Failed to write %s file: %s", path.Join(app.SourceCodePath, "app.yml"), err),
+						Locations: b.Config.GetLocations("resources.apps." + key),
+					})
+					mu.Unlock()
+				}
+				return nil
+			})
+		}
+	}
+
+	if err := errGroup.Wait(); err != nil {
+		return diags.Extend(diag.FromErr(err))
+	}
+
+	return diags
+}
+
+// Name implements bundle.Mutator.
+func (u *uploadConfig) Name() string {
+	return "apps:UploadConfig"
+}
+
+func UploadConfig() bundle.Mutator {
+	return &uploadConfig{
+		filerFactory: func(b *bundle.Bundle) (filer.Filer, error) {
+			return filer.NewWorkspaceFilesClient(b.WorkspaceClient(), b.Config.Workspace.FilePath)
+		},
+	}
+}
+
+func configToYaml(app *resources.App) (*bytes.Buffer, error) {
+	buf := bytes.NewBuffer(nil)
+	enc := yaml.NewEncoder(buf)
+	enc.SetIndent(2)
+
+	err := enc.Encode(app.Config)
+	defer enc.Close()
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to encode app config to yaml: %w", err)
+	}
+
+	return buf, nil
+}
diff --git a/bundle/apps/upload_config_test.go b/bundle/apps/upload_config_test.go
new file mode 100644
index 000000000..a1a6b3afb
--- /dev/null
+++ b/bundle/apps/upload_config_test.go
@@ -0,0 +1,75 @@
+package apps
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/internal/bundletest"
+	mockfiler "github.com/databricks/cli/internal/mocks/libs/filer"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/filer"
+	"github.com/databricks/cli/libs/vfs"
+	"github.com/databricks/databricks-sdk-go/service/apps"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAppUploadConfig(t *testing.T) {
+	root := t.TempDir()
+	err := os.MkdirAll(filepath.Join(root, "my_app"), 0o700)
+	require.NoError(t, err)
+
+	b := &bundle.Bundle{
+		BundleRootPath: root,
+		SyncRootPath:   root,
+		SyncRoot:       vfs.MustNew(root),
+		Config: config.Root{
+			Workspace: config.Workspace{
+				RootPath: "/Workspace/Users/foo@bar.com/",
+			},
+			Resources: config.Resources{
+				Apps: map[string]*resources.App{
+					"my_app": {
+						App: &apps.App{
+							Name: "my_app",
+						},
+						SourceCodePath: "./my_app",
+						Config: map[string]any{
+							"command": []string{"echo", "hello"},
+							"env": []map[string]string{
+								{"name": "MY_APP", "value": "my value"},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	mockFiler := mockfiler.NewMockFiler(t)
+	mockFiler.EXPECT().Write(mock.Anything, "my_app/app.yml", bytes.NewBufferString(`command:
+  - echo
+  - hello
+env:
+  - name: MY_APP
+    value: my value
+`), filer.OverwriteIfExists).Return(nil)
+
+	u := uploadConfig{
+		filerFactory: func(b *bundle.Bundle) (filer.Filer, error) {
+			return mockFiler, nil
+		},
+	}
+
+	bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(root, "databricks.yml")}})
+
+	diags := bundle.Apply(context.Background(), b, bundle.Seq(mutator.TranslatePaths(), &u))
+	require.NoError(t, diags.Error())
+}
diff --git a/bundle/apps/validate.go b/bundle/apps/validate.go
new file mode 100644
index 000000000..fc50aeafc
--- /dev/null
+++ b/bundle/apps/validate.go
@@ -0,0 +1,53 @@
+package apps
+
+import (
+	"context"
+	"fmt"
+	"path"
+	"strings"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+)
+
+type validate struct{}
+
+func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	var diags diag.Diagnostics
+	possibleConfigFiles := []string{"app.yml", "app.yaml"}
+	usedSourceCodePaths := make(map[string]string)
+
+	for key, app := range b.Config.Resources.Apps {
+		if _, ok := usedSourceCodePaths[app.SourceCodePath]; ok {
+			diags = append(diags, diag.Diagnostic{
+				Severity:  diag.Error,
+				Summary:   "Duplicate app source code path",
+				Detail:    fmt.Sprintf("app resource '%s' has the same source code path as app resource '%s', this will lead to the app configuration being overriden by each other", key, usedSourceCodePaths[app.SourceCodePath]),
+				Locations: b.Config.GetLocations(fmt.Sprintf("resources.apps.%s.source_code_path", key)),
+			})
+		}
+		usedSourceCodePaths[app.SourceCodePath] = key
+
+		for _, configFile := range possibleConfigFiles {
+			appPath := strings.TrimPrefix(app.SourceCodePath, b.Config.Workspace.FilePath)
+			cf := path.Join(appPath, configFile)
+			if _, err := b.SyncRoot.Stat(cf); err == nil {
+				diags = append(diags, diag.Diagnostic{
+					Severity: diag.Error,
+					Summary:  configFile + " detected",
+					Detail:   fmt.Sprintf("remove %s and use 'config' property for app resource '%s' instead", cf, app.Name),
+				})
+			}
+		}
+	}
+
+	return diags
+}
+
+func (v *validate) Name() string {
+	return "apps.Validate"
+}
+
+func Validate() bundle.Mutator {
+	return &validate{}
+}
diff --git a/bundle/apps/validate_test.go b/bundle/apps/validate_test.go
new file mode 100644
index 000000000..6c3a88191
--- /dev/null
+++ b/bundle/apps/validate_test.go
@@ -0,0 +1,97 @@
+package apps
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/internal/bundletest"
+	"github.com/databricks/cli/internal/testutil"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/vfs"
+	"github.com/databricks/databricks-sdk-go/service/apps"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAppsValidate(t *testing.T) {
+	tmpDir := t.TempDir()
+	testutil.Touch(t, tmpDir, "app1", "app.yml")
+	testutil.Touch(t, tmpDir, "app2", "app.py")
+
+	b := &bundle.Bundle{
+		BundleRootPath: tmpDir,
+		SyncRootPath:   tmpDir,
+		SyncRoot:       vfs.MustNew(tmpDir),
+		Config: config.Root{
+			Workspace: config.Workspace{
+				FilePath: "/foo/bar/",
+			},
+			Resources: config.Resources{
+				Apps: map[string]*resources.App{
+					"app1": {
+						App: &apps.App{
+							Name: "app1",
+						},
+						SourceCodePath: "./app1",
+					},
+					"app2": {
+						App: &apps.App{
+							Name: "app2",
+						},
+						SourceCodePath: "./app2",
+					},
+				},
+			},
+		},
+	}
+
+	bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}})
+
+	diags := bundle.Apply(context.Background(), b, bundle.Seq(mutator.TranslatePaths(), Validate()))
+	require.Len(t, diags, 1)
+	require.Equal(t, "app.yml detected", diags[0].Summary)
+	require.Contains(t, diags[0].Detail, "app.yml and use 'config' property for app resource")
+}
+
+func TestAppsValidateSameSourcePath(t *testing.T) {
+	tmpDir := t.TempDir()
+	testutil.Touch(t, tmpDir, "app1", "app.py")
+
+	b := &bundle.Bundle{
+		BundleRootPath: tmpDir,
+		SyncRootPath:   tmpDir,
+		SyncRoot:       vfs.MustNew(tmpDir),
+		Config: config.Root{
+			Workspace: config.Workspace{
+				FilePath: "/foo/bar/",
+			},
+			Resources: config.Resources{
+				Apps: map[string]*resources.App{
+					"app1": {
+						App: &apps.App{
+							Name: "app1",
+						},
+						SourceCodePath: "./app1",
+					},
+					"app2": {
+						App: &apps.App{
+							Name: "app2",
+						},
+						SourceCodePath: "./app1",
+					},
+				},
+			},
+		},
+	}
+
+	bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}})
+
+	diags := bundle.Apply(context.Background(), b, bundle.Seq(mutator.TranslatePaths(), Validate()))
+	require.Len(t, diags, 1)
+	require.Equal(t, "Duplicate app source code path", diags[0].Summary)
+	require.Contains(t, diags[0].Detail, "has the same source code path as app resource")
+}
diff --git a/bundle/config/generate/app.go b/bundle/config/generate/app.go
new file mode 100644
index 000000000..1255d63f8
--- /dev/null
+++ b/bundle/config/generate/app.go
@@ -0,0 +1,37 @@
+package generate
+
+import (
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/databricks/databricks-sdk-go/service/apps"
+)
+
+func ConvertAppToValue(app *apps.App, sourceCodePath string, appConfig map[string]any) (dyn.Value, error) {
+	ac, err := convert.FromTyped(appConfig, dyn.NilValue)
+	if err != nil {
+		return dyn.NilValue, err
+	}
+
+	ar, err := convert.FromTyped(app.Resources, dyn.NilValue)
+	if err != nil {
+		return dyn.NilValue, err
+	}
+
+	// The majority of fields of the app struct are read-only.
+	// We copy the relevant fields manually.
+	dv := map[string]dyn.Value{
+		"name":             dyn.NewValue(app.Name, []dyn.Location{{Line: 1}}),
+		"description":      dyn.NewValue(app.Description, []dyn.Location{{Line: 2}}),
+		"source_code_path": dyn.NewValue(sourceCodePath, []dyn.Location{{Line: 3}}),
+	}
+
+	if ac.Kind() != dyn.KindNil {
+		dv["config"] = ac.WithLocations([]dyn.Location{{Line: 4}})
+	}
+
+	if ar.Kind() != dyn.KindNil {
+		dv["resources"] = ar.WithLocations([]dyn.Location{{Line: 5}})
+	}
+
+	return dyn.V(dv), nil
+}
diff --git a/bundle/config/mutator/apply_presets.go b/bundle/config/mutator/apply_presets.go
index 59b8547be..b402053e7 100644
--- a/bundle/config/mutator/apply_presets.go
+++ b/bundle/config/mutator/apply_presets.go
@@ -221,6 +221,8 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 		dashboard.DisplayName = prefix + dashboard.DisplayName
 	}
 
+	// Apps: No presets
+
 	return diags
 }
 
diff --git a/bundle/config/mutator/apply_source_linked_deployment_preset.go b/bundle/config/mutator/apply_source_linked_deployment_preset.go
index 78ccc5322..839648301 100644
--- a/bundle/config/mutator/apply_source_linked_deployment_preset.go
+++ b/bundle/config/mutator/apply_source_linked_deployment_preset.go
@@ -56,6 +56,22 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle
 		b.Config.Presets.SourceLinkedDeployment = &enabled
 	}
 
+	if len(b.Config.Resources.Apps) > 0 && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
+		path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment"))
+		diags = diags.Append(
+			diag.Diagnostic{
+				Severity: diag.Error,
+				Summary:  "source-linked deployment is not supported for apps",
+				Paths: []dyn.Path{
+					path,
+				},
+				Locations: b.Config.GetLocations(path[2:].String()),
+			},
+		)
+
+		return diags
+	}
+
 	if b.Config.Workspace.FilePath != "" && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
 		path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("workspace"), dyn.Key("file_path"))
 
diff --git a/bundle/config/mutator/apply_source_linked_deployment_preset_test.go b/bundle/config/mutator/apply_source_linked_deployment_preset_test.go
index 1b74fd8e9..42fda8ea7 100644
--- a/bundle/config/mutator/apply_source_linked_deployment_preset_test.go
+++ b/bundle/config/mutator/apply_source_linked_deployment_preset_test.go
@@ -8,6 +8,7 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/cli/bundle/config/resources"
 	"github.com/databricks/cli/bundle/internal/bundletest"
 	"github.com/databricks/cli/libs/dbr"
 	"github.com/databricks/cli/libs/dyn"
@@ -31,6 +32,7 @@ func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
 		initialValue    *bool
 		expectedValue   *bool
 		expectedWarning string
+		expectedError   string
 	}{
 		{
 			name:          "preset enabled, bundle in Workspace, databricks runtime",
@@ -86,6 +88,18 @@ func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
 			expectedValue:   &enabled,
 			expectedWarning: "workspace.file_path setting will be ignored in source-linked deployment mode",
 		},
+		{
+			name: "preset enabled, apps is defined by user",
+			ctx:  dbr.MockRuntime(testContext, true),
+			mutateBundle: func(b *bundle.Bundle) {
+				b.Config.Resources.Apps = map[string]*resources.App{
+					"app": {},
+				}
+			},
+			initialValue:  &enabled,
+			expectedValue: &enabled,
+			expectedError: "source-linked deployment is not supported for apps",
+		},
 	}
 
 	for _, tt := range tests {
@@ -107,7 +121,7 @@ func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
 			bundletest.SetLocation(b, "workspace.file_path", []dyn.Location{{File: "databricks.yml"}})
 
 			diags := bundle.Apply(tt.ctx, b, mutator.ApplySourceLinkedDeploymentPreset())
-			if diags.HasError() {
+			if diags.HasError() && tt.expectedError == "" {
 				t.Fatalf("unexpected error: %v", diags)
 			}
 
@@ -116,6 +130,11 @@ func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
 				require.NotEmpty(t, diags[0].Locations)
 			}
 
+			if tt.expectedError != "" {
+				require.Equal(t, tt.expectedError, diags[0].Summary)
+				require.NotEmpty(t, diags[0].Locations)
+			}
+
 			require.Equal(t, tt.expectedValue, b.Config.Presets.SourceLinkedDeployment)
 		})
 	}
diff --git a/bundle/config/mutator/merge_apps.go b/bundle/config/mutator/merge_apps.go
new file mode 100644
index 000000000..d91e8dd7f
--- /dev/null
+++ b/bundle/config/mutator/merge_apps.go
@@ -0,0 +1,45 @@
+package mutator
+
+import (
+	"context"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/merge"
+)
+
+type mergeApps struct{}
+
+func MergeApps() bundle.Mutator {
+	return &mergeApps{}
+}
+
+func (m *mergeApps) Name() string {
+	return "MergeApps"
+}
+
+func (m *mergeApps) resourceName(v dyn.Value) string {
+	switch v.Kind() {
+	case dyn.KindInvalid, dyn.KindNil:
+		return ""
+	case dyn.KindString:
+		return v.MustString()
+	default:
+		panic("app name must be a string")
+	}
+}
+
+func (m *mergeApps) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
+		if v.Kind() == dyn.KindNil {
+			return v, nil
+		}
+
+		return dyn.Map(v, "resources.apps", dyn.Foreach(func(_ dyn.Path, app dyn.Value) (dyn.Value, error) {
+			return dyn.Map(app, "resources", merge.ElementsByKeyWithOverride("name", m.resourceName))
+		}))
+	})
+
+	return diag.FromErr(err)
+}
diff --git a/bundle/config/mutator/merge_apps_test.go b/bundle/config/mutator/merge_apps_test.go
new file mode 100644
index 000000000..0a161b845
--- /dev/null
+++ b/bundle/config/mutator/merge_apps_test.go
@@ -0,0 +1,73 @@
+package mutator_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/databricks-sdk-go/service/apps"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMergeApps(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Apps: map[string]*resources.App{
+					"foo": {
+						App: &apps.App{
+							Name: "foo",
+							Resources: []apps.AppResource{
+								{
+									Name: "job1",
+									Job: &apps.AppResourceJob{
+										Id:         "1234",
+										Permission: "CAN_MANAGE_RUN",
+									},
+								},
+								{
+									Name: "sql1",
+									SqlWarehouse: &apps.AppResourceSqlWarehouse{
+										Id:         "5678",
+										Permission: "CAN_USE",
+									},
+								},
+								{
+									Name: "job1",
+									Job: &apps.AppResourceJob{
+										Id:         "1234",
+										Permission: "CAN_MANAGE",
+									},
+								},
+								{
+									Name: "sql1",
+									Job: &apps.AppResourceJob{
+										Id:         "9876",
+										Permission: "CAN_MANAGE",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	diags := bundle.Apply(context.Background(), b, mutator.MergeApps())
+	assert.NoError(t, diags.Error())
+
+	j := b.Config.Resources.Apps["foo"]
+
+	assert.Len(t, j.Resources, 2)
+	assert.Equal(t, "job1", j.Resources[0].Name)
+	assert.Equal(t, "sql1", j.Resources[1].Name)
+
+	assert.Equal(t, "CAN_MANAGE", string(j.Resources[0].Job.Permission))
+
+	assert.Nil(t, j.Resources[1].SqlWarehouse)
+	assert.Equal(t, "CAN_MANAGE", string(j.Resources[1].Job.Permission))
+}
diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go
index 6df88d067..723b01ee3 100644
--- a/bundle/config/mutator/process_target_mode_test.go
+++ b/bundle/config/mutator/process_target_mode_test.go
@@ -13,6 +13,7 @@ import (
 	"github.com/databricks/cli/libs/tags"
 	"github.com/databricks/cli/libs/vfs"
 	sdkconfig "github.com/databricks/databricks-sdk-go/config"
+	"github.com/databricks/databricks-sdk-go/service/apps"
 	"github.com/databricks/databricks-sdk-go/service/catalog"
 	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/dashboards"
@@ -142,6 +143,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
 						},
 					},
 				},
+				Apps: map[string]*resources.App{
+					"app1": {
+						App: &apps.App{
+							Name: "app1",
+						},
+					},
+				},
 			},
 		},
 		SyncRoot: vfs.MustNew("/Users/lennart.kats@databricks.com"),
@@ -433,6 +441,13 @@ func TestAllNonUcResourcesAreRenamed(t *testing.T) {
 			for _, key := range field.MapKeys() {
 				resource := field.MapIndex(key)
 				nameField := resource.Elem().FieldByName("Name")
+				resourceType := resources.Type().Field(i).Name
+
+				// Skip apps, as they are not renamed
+				if resourceType == "Apps" {
+					continue
+				}
+
 				if !nameField.IsValid() || nameField.Kind() != reflect.String {
 					continue
 				}
diff --git a/bundle/config/mutator/run_as.go b/bundle/config/mutator/run_as.go
index 7ffd782c2..3d7391b01 100644
--- a/bundle/config/mutator/run_as.go
+++ b/bundle/config/mutator/run_as.go
@@ -119,6 +119,16 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics {
 		))
 	}
 
+	// Apps do not support run_as in the API.
+	if len(b.Config.Resources.Apps) > 0 {
+		diags = diags.Extend(reportRunAsNotSupported(
+			"apps",
+			b.Config.GetLocation("resources.apps"),
+			b.Config.Workspace.CurrentUser.UserName,
+			identity,
+		))
+	}
+
 	return diags
 }
 
diff --git a/bundle/config/mutator/run_as_test.go b/bundle/config/mutator/run_as_test.go
index dbf4bf806..650b65d61 100644
--- a/bundle/config/mutator/run_as_test.go
+++ b/bundle/config/mutator/run_as_test.go
@@ -32,6 +32,7 @@ func allResourceTypes(t *testing.T) []string {
 	// the dyn library gives us the correct list of all resources supported. Please
 	// also update this check when adding a new resource
 	require.Equal(t, []string{
+		"apps",
 		"clusters",
 		"dashboards",
 		"experiments",
@@ -104,47 +105,47 @@ func TestRunAsWorksForAllowedResources(t *testing.T) {
 	}
 }
 
-func TestRunAsErrorForUnsupportedResources(t *testing.T) {
-	// Bundle "run_as" has two modes of operation, each with a different set of
-	// resources that are supported.
-	// Cases:
-	//   1. When the bundle "run_as" identity is same as the current deployment
-	//      identity. In this case all resources are supported.
-	//   2. When the bundle "run_as" identity is different from the current
-	//      deployment identity. In this case only a subset of resources are
-	//      supported. This subset of resources are defined in the allow list below.
-	//
-	// To be a part of the allow list, the resource must satisfy one of the following
-	// two conditions:
-	//   1. The resource supports setting a run_as identity to a different user
-	//      from the owner/creator of the resource. For example, jobs.
-	//   2. Run as semantics do not apply to the resource. We do not plan to add
-	//      platform side support for `run_as` for these resources. For example,
-	//      experiments or registered models.
-	//
-	// Any resource that is not on the allow list cannot be used when the bundle
-	// run_as is different from the current deployment user. "bundle validate" must
-	// return an error if such a resource has been defined, and the run_as identity
-	// is different from the current deployment identity.
-	//
-	// Action Item: If you are adding a new resource to DABs, please check in with
-	// the relevant owning team whether the resource should be on the allow list or (implicitly) on
-	// the deny list. Any resources that could have run_as semantics in the future
-	// should be on the deny list.
-	// For example: Teams for pipelines, model serving endpoints or Lakeview dashboards
-	// are planning to add platform side support for `run_as` for these resources at
-	// some point in the future. These resources are (implicitly) on the deny list, since
-	// they are not on the allow list below.
-	allowList := []string{
-		"clusters",
-		"jobs",
-		"models",
-		"registered_models",
-		"experiments",
-		"schemas",
-		"volumes",
-	}
+// Bundle "run_as" has two modes of operation, each with a different set of
+// resources that are supported.
+// Cases:
+//  1. When the bundle "run_as" identity is same as the current deployment
+//     identity. In this case all resources are supported.
+//  2. When the bundle "run_as" identity is different from the current
+//     deployment identity. In this case only a subset of resources are
+//     supported. This subset of resources are defined in the allow list below.
+//
+// To be a part of the allow list, the resource must satisfy one of the following
+// two conditions:
+//  1. The resource supports setting a run_as identity to a different user
+//     from the owner/creator of the resource. For example, jobs.
+//  2. Run as semantics do not apply to the resource. We do not plan to add
+//     platform side support for `run_as` for these resources. For example,
+//     experiments or registered models.
+//
+// Any resource that is not on the allow list cannot be used when the bundle
+// run_as is different from the current deployment user. "bundle validate" must
+// return an error if such a resource has been defined, and the run_as identity
+// is different from the current deployment identity.
+//
+// Action Item: If you are adding a new resource to DABs, please check in with
+// the relevant owning team whether the resource should be on the allow list or (implicitly) on
+// the deny list. Any resources that could have run_as semantics in the future
+// should be on the deny list.
+// For example: Teams for pipelines, model serving endpoints or Lakeview dashboards
+// are planning to add platform side support for `run_as` for these resources at
+// some point in the future. These resources are (implicitly) on the deny list, since
+// they are not on the allow list below.
+var allowList = []string{
+	"clusters",
+	"jobs",
+	"models",
+	"registered_models",
+	"experiments",
+	"schemas",
+	"volumes",
+}
 
+func TestRunAsErrorForUnsupportedResources(t *testing.T) {
 	base := config.Root{
 		Workspace: config.Workspace{
 			CurrentUser: &config.User{
@@ -197,3 +198,54 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) {
 			"See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.", rt)
 	}
 }
+
+func TestRunAsNoErrorForSupportedResources(t *testing.T) {
+	base := config.Root{
+		Workspace: config.Workspace{
+			CurrentUser: &config.User{
+				User: &iam.User{
+					UserName: "alice",
+				},
+			},
+		},
+		RunAs: &jobs.JobRunAs{
+			UserName: "bob",
+		},
+	}
+
+	v, err := convert.FromTyped(base, dyn.NilValue)
+	require.NoError(t, err)
+
+	// Define top level resources key in the bundle configuration.
+	// This is not part of the typed configuration, so we need to add it manually.
+	v, err = dyn.Set(v, "resources", dyn.V(map[string]dyn.Value{}))
+	require.NoError(t, err)
+
+	for _, rt := range allResourceTypes(t) {
+		// Skip unsupported resources
+		if !slices.Contains(allowList, rt) {
+			continue
+		}
+
+		// Add an instance of the resource type that is not on the allow list to
+		// the bundle configuration.
+		nv, err := dyn.SetByPath(v, dyn.NewPath(dyn.Key("resources"), dyn.Key(rt)), dyn.V(map[string]dyn.Value{
+			"foo": dyn.V(map[string]dyn.Value{
+				"name": dyn.V("bar"),
+			}),
+		}))
+		require.NoError(t, err)
+
+		// Get back typed configuration from the newly created invalid bundle configuration.
+		r := &config.Root{}
+		err = convert.ToTyped(r, nv)
+		require.NoError(t, err)
+
+		// Assert this configuration passes validation.
+		b := &bundle.Bundle{
+			Config: *r,
+		}
+		diags := bundle.Apply(context.Background(), b, SetRunAs())
+		require.NoError(t, diags.Error())
+	}
+}
diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go
index af0f94120..1915cf36e 100644
--- a/bundle/config/mutator/translate_paths.go
+++ b/bundle/config/mutator/translate_paths.go
@@ -262,6 +262,7 @@ func (m *translatePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos
 			t.applyPipelineTranslations,
 			t.applyArtifactTranslations,
 			t.applyDashboardTranslations,
+			t.applyAppsTranslations,
 		} {
 			v, err = fn(v)
 			if err != nil {
diff --git a/bundle/config/mutator/translate_paths_apps.go b/bundle/config/mutator/translate_paths_apps.go
new file mode 100644
index 000000000..0ed7e1928
--- /dev/null
+++ b/bundle/config/mutator/translate_paths_apps.go
@@ -0,0 +1,28 @@
+package mutator
+
+import (
+	"fmt"
+
+	"github.com/databricks/cli/libs/dyn"
+)
+
+func (t *translateContext) applyAppsTranslations(v dyn.Value) (dyn.Value, error) {
+	// Convert the `source_code_path` field to a remote absolute path.
+	// We use this path for app deployment to point to the source code.
+	pattern := dyn.NewPattern(
+		dyn.Key("resources"),
+		dyn.Key("apps"),
+		dyn.AnyKey(),
+		dyn.Key("source_code_path"),
+	)
+
+	return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+		key := p[2].Key()
+		dir, err := v.Location().Directory()
+		if err != nil {
+			return dyn.InvalidValue, fmt.Errorf("unable to determine directory for app %s: %w", key, err)
+		}
+
+		return t.rewriteRelativeTo(p, v, t.translateDirectoryPath, dir, "")
+	})
+}
diff --git a/bundle/config/mutator/translate_paths_apps_test.go b/bundle/config/mutator/translate_paths_apps_test.go
new file mode 100644
index 000000000..5692934b8
--- /dev/null
+++ b/bundle/config/mutator/translate_paths_apps_test.go
@@ -0,0 +1,57 @@
+package mutator_test
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/internal/bundletest"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/vfs"
+	"github.com/databricks/databricks-sdk-go/service/apps"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTranslatePathsApps_FilePathRelativeSubDirectory(t *testing.T) {
+	dir := t.TempDir()
+	touchEmptyFile(t, filepath.Join(dir, "src", "app", "app.py"))
+
+	b := &bundle.Bundle{
+		SyncRootPath: dir,
+		SyncRoot:     vfs.MustNew(dir),
+		Config: config.Root{
+			Workspace: config.Workspace{
+				FilePath: "/bundle/files",
+			},
+			Resources: config.Resources{
+				Apps: map[string]*resources.App{
+					"app": {
+						App: &apps.App{
+							Name: "My App",
+						},
+						SourceCodePath: "../src/app",
+					},
+				},
+			},
+		},
+	}
+
+	bundletest.SetLocation(b, "resources.apps", []dyn.Location{{
+		File: filepath.Join(dir, "resources/app.yml"),
+	}})
+
+	diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
+	require.NoError(t, diags.Error())
+
+	// Assert that the file path for the app has been converted to its local absolute path.
+	assert.Equal(
+		t,
+		"/bundle/files/src/app",
+		b.Config.Resources.Apps["app"].SourceCodePath,
+	)
+}
diff --git a/bundle/config/resources.go b/bundle/config/resources.go
index 13cf0d462..1f523fed3 100644
--- a/bundle/config/resources.go
+++ b/bundle/config/resources.go
@@ -23,6 +23,7 @@ type Resources struct {
 	Volumes               map[string]*resources.Volume               `json:"volumes,omitempty"`
 	Clusters              map[string]*resources.Cluster              `json:"clusters,omitempty"`
 	Dashboards            map[string]*resources.Dashboard            `json:"dashboards,omitempty"`
+	Apps                  map[string]*resources.App                  `json:"apps,omitempty"`
 }
 
 type ConfigResource interface {
@@ -87,6 +88,7 @@ func (r *Resources) AllResources() []ResourceGroup {
 		collectResourceMap(descriptions["clusters"], r.Clusters),
 		collectResourceMap(descriptions["dashboards"], r.Dashboards),
 		collectResourceMap(descriptions["volumes"], r.Volumes),
+		collectResourceMap(descriptions["apps"], r.Apps),
 	}
 }
 
@@ -97,12 +99,19 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error)
 			found = append(found, r.Jobs[k])
 		}
 	}
+
 	for k := range r.Pipelines {
 		if k == key {
 			found = append(found, r.Pipelines[k])
 		}
 	}
 
+	for k := range r.Apps {
+		if k == key {
+			found = append(found, r.Apps[k])
+		}
+	}
+
 	if len(found) == 0 {
 		return nil, fmt.Errorf("no such resource: %s", key)
 	}
@@ -126,76 +135,96 @@ type ResourceDescription struct {
 	// Singular and plural title when used in summaries / terminal UI.
 	SingularTitle string
 	PluralTitle   string
+
+	TerraformResourceName string
 }
 
 // The keys of the map corresponds to the resource key in the bundle configuration.
 func SupportedResources() map[string]ResourceDescription {
 	return map[string]ResourceDescription{
 		"jobs": {
-			SingularName:  "job",
-			PluralName:    "jobs",
-			SingularTitle: "Job",
-			PluralTitle:   "Jobs",
+			SingularName:          "job",
+			PluralName:            "jobs",
+			SingularTitle:         "Job",
+			PluralTitle:           "Jobs",
+			TerraformResourceName: "databricks_job",
 		},
 		"pipelines": {
-			SingularName:  "pipeline",
-			PluralName:    "pipelines",
-			SingularTitle: "Pipeline",
-			PluralTitle:   "Pipelines",
+			SingularName:          "pipeline",
+			PluralName:            "pipelines",
+			SingularTitle:         "Pipeline",
+			PluralTitle:           "Pipelines",
+			TerraformResourceName: "databricks_pipeline",
 		},
 		"models": {
-			SingularName:  "model",
-			PluralName:    "models",
-			SingularTitle: "Model",
-			PluralTitle:   "Models",
+			SingularName:          "model",
+			PluralName:            "models",
+			SingularTitle:         "Model",
+			PluralTitle:           "Models",
+			TerraformResourceName: "databricks_mlflow_model",
 		},
 		"experiments": {
-			SingularName:  "experiment",
-			PluralName:    "experiments",
-			SingularTitle: "Experiment",
-			PluralTitle:   "Experiments",
+			SingularName:          "experiment",
+			PluralName:            "experiments",
+			SingularTitle:         "Experiment",
+			PluralTitle:           "Experiments",
+			TerraformResourceName: "databricks_mlflow_experiment",
 		},
 		"model_serving_endpoints": {
-			SingularName:  "model_serving_endpoint",
-			PluralName:    "model_serving_endpoints",
-			SingularTitle: "Model Serving Endpoint",
-			PluralTitle:   "Model Serving Endpoints",
+			SingularName:          "model_serving_endpoint",
+			PluralName:            "model_serving_endpoints",
+			SingularTitle:         "Model Serving Endpoint",
+			PluralTitle:           "Model Serving Endpoints",
+			TerraformResourceName: "databricks_model_serving_endpoint",
 		},
 		"registered_models": {
-			SingularName:  "registered_model",
-			PluralName:    "registered_models",
-			SingularTitle: "Registered Model",
-			PluralTitle:   "Registered Models",
+			SingularName:          "registered_model",
+			PluralName:            "registered_models",
+			SingularTitle:         "Registered Model",
+			PluralTitle:           "Registered Models",
+			TerraformResourceName: "databricks_registered_model",
 		},
 		"quality_monitors": {
-			SingularName:  "quality_monitor",
-			PluralName:    "quality_monitors",
-			SingularTitle: "Quality Monitor",
-			PluralTitle:   "Quality Monitors",
+			SingularName:          "quality_monitor",
+			PluralName:            "quality_monitors",
+			SingularTitle:         "Quality Monitor",
+			PluralTitle:           "Quality Monitors",
+			TerraformResourceName: "databricks_quality_monitor",
 		},
 		"schemas": {
-			SingularName:  "schema",
-			PluralName:    "schemas",
-			SingularTitle: "Schema",
-			PluralTitle:   "Schemas",
+			SingularName:          "schema",
+			PluralName:            "schemas",
+			SingularTitle:         "Schema",
+			PluralTitle:           "Schemas",
+			TerraformResourceName: "databricks_schema",
 		},
 		"clusters": {
-			SingularName:  "cluster",
-			PluralName:    "clusters",
-			SingularTitle: "Cluster",
-			PluralTitle:   "Clusters",
+			SingularName:          "cluster",
+			PluralName:            "clusters",
+			SingularTitle:         "Cluster",
+			PluralTitle:           "Clusters",
+			TerraformResourceName: "databricks_cluster",
 		},
 		"dashboards": {
-			SingularName:  "dashboard",
-			PluralName:    "dashboards",
-			SingularTitle: "Dashboard",
-			PluralTitle:   "Dashboards",
+			SingularName:          "dashboard",
+			PluralName:            "dashboards",
+			SingularTitle:         "Dashboard",
+			PluralTitle:           "Dashboards",
+			TerraformResourceName: "databricks_dashboard",
 		},
 		"volumes": {
-			SingularName:  "volume",
-			PluralName:    "volumes",
-			SingularTitle: "Volume",
-			PluralTitle:   "Volumes",
+			SingularName:          "volume",
+			PluralName:            "volumes",
+			SingularTitle:         "Volume",
+			PluralTitle:           "Volumes",
+			TerraformResourceName: "databricks_volume",
+		},
+		"apps": {
+			SingularName:          "app",
+			PluralName:            "apps",
+			SingularTitle:         "App",
+			PluralTitle:           "Apps",
+			TerraformResourceName: "databricks_app",
 		},
 	}
 }
diff --git a/bundle/config/resources/apps.go b/bundle/config/resources/apps.go
new file mode 100644
index 000000000..809e04896
--- /dev/null
+++ b/bundle/config/resources/apps.go
@@ -0,0 +1,70 @@
+package resources
+
+import (
+	"context"
+	"net/url"
+
+	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/marshal"
+	"github.com/databricks/databricks-sdk-go/service/apps"
+)
+
+type App struct {
+	// SourceCodePath is a required field used by DABs to point to Databricks app source code
+	// on local disk and to the corresponding workspace path during app deployment.
+	SourceCodePath string `json:"source_code_path"`
+
+	// Config is an optional field which allows configuring the app following Databricks app configuration format like in app.yml.
+	// When this field is set, DABs read the configuration set in this field and write
+	// it to app.yml in the root of the source code folder in Databricks workspace.
+	// If there’s app.yml defined locally, DABs will raise an error.
+	Config map[string]any `json:"config,omitempty"`
+
+	Permissions    []Permission   `json:"permissions,omitempty"`
+	ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
+	URL            string         `json:"url,omitempty" bundle:"internal"`
+
+	*apps.App
+}
+
+func (a *App) UnmarshalJSON(b []byte) error {
+	return marshal.Unmarshal(b, a)
+}
+
+func (a App) MarshalJSON() ([]byte, error) {
+	return marshal.Marshal(a)
+}
+
+func (a *App) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) {
+	_, err := w.Apps.GetByName(ctx, name)
+	if err != nil {
+		log.Debugf(ctx, "app %s does not exist", name)
+		return false, err
+	}
+	return true, nil
+}
+
+func (a *App) TerraformResourceName() string {
+	return "databricks_app"
+}
+
+func (a *App) InitializeURL(baseURL url.URL) {
+	if a.ModifiedStatus == "" || a.ModifiedStatus == ModifiedStatusCreated {
+		return
+	}
+	baseURL.Path = "apps/" + a.Name
+	a.URL = baseURL.String()
+}
+
+func (a *App) GetName() string {
+	return a.Name
+}
+
+func (a *App) GetURL() string {
+	return a.URL
+}
+
+func (a *App) IsNil() bool {
+	return a.App == nil
+}
diff --git a/bundle/deploy/terraform/convert.go b/bundle/deploy/terraform/convert.go
index b710c690f..d549b9797 100644
--- a/bundle/deploy/terraform/convert.go
+++ b/bundle/deploy/terraform/convert.go
@@ -9,6 +9,7 @@ import (
 	"github.com/databricks/cli/bundle/deploy/terraform/tfdyn"
 	"github.com/databricks/cli/bundle/internal/tf/schema"
 	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/databricks-sdk-go/service/apps"
 	tfjson "github.com/hashicorp/terraform-json"
 )
 
@@ -196,6 +197,20 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error {
 				}
 				cur.ID = instance.Attributes.ID
 				config.Resources.Dashboards[resource.Name] = cur
+			case "databricks_app":
+				if config.Resources.Apps == nil {
+					config.Resources.Apps = make(map[string]*resources.App)
+				}
+				cur := config.Resources.Apps[resource.Name]
+				if cur == nil {
+					cur = &resources.App{ModifiedStatus: resources.ModifiedStatusDeleted, App: &apps.App{}}
+				} else {
+					// If the app exists in terraform and bundle, we always set modified status to updated
+					// because we don't really know if the app source code was updated or not.
+					cur.ModifiedStatus = resources.ModifiedStatusUpdated
+				}
+				cur.Name = instance.Attributes.Name
+				config.Resources.Apps[resource.Name] = cur
 			case "databricks_permissions":
 			case "databricks_grants":
 				// Ignore; no need to pull these back into the configuration.
@@ -260,6 +275,11 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error {
 			src.ModifiedStatus = resources.ModifiedStatusCreated
 		}
 	}
+	for _, src := range config.Resources.Apps {
+		if src.ModifiedStatus == "" {
+			src.ModifiedStatus = resources.ModifiedStatusCreated
+		}
+	}
 
 	return nil
 }
diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go
index ccfdcece3..ffe55db71 100644
--- a/bundle/deploy/terraform/convert_test.go
+++ b/bundle/deploy/terraform/convert_test.go
@@ -10,6 +10,7 @@ import (
 	"github.com/databricks/cli/bundle/internal/tf/schema"
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/databricks/databricks-sdk-go/service/apps"
 	"github.com/databricks/databricks-sdk-go/service/catalog"
 	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/dashboards"
@@ -694,6 +695,14 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) {
 					{Attributes: stateInstanceAttributes{ID: "1"}},
 				},
 			},
+			{
+				Type: "databricks_app",
+				Mode: "managed",
+				Name: "test_app",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{Name: "app1"}},
+				},
+			},
 		},
 	}
 	err := TerraformToBundle(&tfState, &config)
@@ -732,6 +741,9 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) {
 	assert.Equal(t, "1", config.Resources.Dashboards["test_dashboard"].ID)
 	assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Dashboards["test_dashboard"].ModifiedStatus)
 
+	assert.Equal(t, "app1", config.Resources.Apps["test_app"].Name)
+	assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Apps["test_app"].ModifiedStatus)
+
 	AssertFullResourceCoverage(t, &config)
 }
 
@@ -815,6 +827,13 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
 					},
 				},
 			},
+			Apps: map[string]*resources.App{
+				"test_app": {
+					App: &apps.App{
+						Description: "test_app",
+					},
+				},
+			},
 		},
 	}
 	tfState := resourcesState{
@@ -856,6 +875,9 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
 	assert.Equal(t, "", config.Resources.Dashboards["test_dashboard"].ID)
 	assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard"].ModifiedStatus)
 
+	assert.Equal(t, "", config.Resources.Apps["test_app"].Name)
+	assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Apps["test_app"].ModifiedStatus)
+
 	AssertFullResourceCoverage(t, &config)
 }
 
@@ -994,6 +1016,18 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
 					},
 				},
 			},
+			Apps: map[string]*resources.App{
+				"test_app": {
+					App: &apps.App{
+						Name: "test_app",
+					},
+				},
+				"test_app_new": {
+					App: &apps.App{
+						Name: "test_app_new",
+					},
+				},
+			},
 		},
 	}
 	tfState := resourcesState{
@@ -1174,6 +1208,22 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
 					{Attributes: stateInstanceAttributes{ID: "2"}},
 				},
 			},
+			{
+				Type: "databricks_app",
+				Mode: "managed",
+				Name: "test_app",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{Name: "test_app"}},
+				},
+			},
+			{
+				Type: "databricks_app",
+				Mode: "managed",
+				Name: "test_app_old",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{Name: "test_app_old"}},
+				},
+			},
 		},
 	}
 	err := TerraformToBundle(&tfState, &config)
@@ -1256,6 +1306,13 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
 	assert.Equal(t, "", config.Resources.Dashboards["test_dashboard_new"].ID)
 	assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard_new"].ModifiedStatus)
 
+	assert.Equal(t, "test_app", config.Resources.Apps["test_app"].Name)
+	assert.Equal(t, resources.ModifiedStatusUpdated, config.Resources.Apps["test_app"].ModifiedStatus)
+	assert.Equal(t, "test_app_old", config.Resources.Apps["test_app_old"].Name)
+	assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Apps["test_app_old"].ModifiedStatus)
+	assert.Equal(t, "test_app_new", config.Resources.Apps["test_app_new"].Name)
+	assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Apps["test_app_new"].ModifiedStatus)
+
 	AssertFullResourceCoverage(t, &config)
 }
 
diff --git a/bundle/deploy/terraform/interpolate.go b/bundle/deploy/terraform/interpolate.go
index 813e6bbb7..719e6ad25 100644
--- a/bundle/deploy/terraform/interpolate.go
+++ b/bundle/deploy/terraform/interpolate.go
@@ -63,6 +63,8 @@ func (m *interpolateMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.D
 				path = dyn.NewPath(dyn.Key("databricks_cluster")).Append(path[2:]...)
 			case dyn.Key("dashboards"):
 				path = dyn.NewPath(dyn.Key("databricks_dashboard")).Append(path[2:]...)
+			case dyn.Key("apps"):
+				path = dyn.NewPath(dyn.Key("databricks_app")).Append(path[2:]...)
 			default:
 				// Trigger "key not found" for unknown resource types.
 				return dyn.GetByPath(root, path)
diff --git a/bundle/deploy/terraform/interpolate_test.go b/bundle/deploy/terraform/interpolate_test.go
index fc5c4d184..91a7bd54a 100644
--- a/bundle/deploy/terraform/interpolate_test.go
+++ b/bundle/deploy/terraform/interpolate_test.go
@@ -34,6 +34,7 @@ func TestInterpolate(t *testing.T) {
 								"other_volume":           "${resources.volumes.other_volume.id}",
 								"other_cluster":          "${resources.clusters.other_cluster.id}",
 								"other_dashboard":        "${resources.dashboards.other_dashboard.id}",
+								"other_app":              "${resources.apps.other_app.id}",
 							},
 							Tasks: []jobs.Task{
 								{
@@ -73,6 +74,7 @@ func TestInterpolate(t *testing.T) {
 	assert.Equal(t, "${databricks_volume.other_volume.id}", j.Tags["other_volume"])
 	assert.Equal(t, "${databricks_cluster.other_cluster.id}", j.Tags["other_cluster"])
 	assert.Equal(t, "${databricks_dashboard.other_dashboard.id}", j.Tags["other_dashboard"])
+	assert.Equal(t, "${databricks_app.other_app.id}", j.Tags["other_app"])
 
 	m := b.Config.Resources.Models["my_model"]
 	assert.Equal(t, "my_model", m.Model.Name)
diff --git a/bundle/deploy/terraform/tfdyn/convert_app.go b/bundle/deploy/terraform/tfdyn/convert_app.go
new file mode 100644
index 000000000..dcba0809b
--- /dev/null
+++ b/bundle/deploy/terraform/tfdyn/convert_app.go
@@ -0,0 +1,55 @@
+package tfdyn
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/cli/bundle/internal/tf/schema"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/databricks-sdk-go/service/apps"
+)
+
+func convertAppResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) {
+	// Check if the description is not set and if it's not, set it to an empty string.
+	// This is done to avoid TF drift because Apps API return empty string for description when if it's not set.
+	if _, err := dyn.Get(vin, "description"); err != nil {
+		vin, err = dyn.Set(vin, "description", dyn.V(""))
+		if err != nil {
+			return vin, err
+		}
+	}
+
+	// Normalize the output value to the target schema.
+	vout, diags := convert.Normalize(apps.App{}, vin)
+	for _, diag := range diags {
+		log.Debugf(ctx, "app normalization diagnostic: %s", diag.Summary)
+	}
+
+	return vout, nil
+}
+
+type appConverter struct{}
+
+func (appConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error {
+	vout, err := convertAppResource(ctx, vin)
+	if err != nil {
+		return err
+	}
+
+	// Add the converted resource to the output.
+	out.App[key] = vout.AsAny()
+
+	// Configure permissions for this resource.
+	if permissions := convertPermissionsResource(ctx, vin); permissions != nil {
+		permissions.AppName = fmt.Sprintf("${databricks_app.%s.name}", key)
+		out.Permissions["app_"+key] = permissions
+	}
+
+	return nil
+}
+
+func init() {
+	registerConverter("apps", appConverter{})
+}
diff --git a/bundle/deploy/terraform/tfdyn/convert_app_test.go b/bundle/deploy/terraform/tfdyn/convert_app_test.go
new file mode 100644
index 000000000..be8152cc6
--- /dev/null
+++ b/bundle/deploy/terraform/tfdyn/convert_app_test.go
@@ -0,0 +1,156 @@
+package tfdyn
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/internal/tf/schema"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/databricks/databricks-sdk-go/service/apps"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConvertApp(t *testing.T) {
+	src := resources.App{
+		SourceCodePath: "./app",
+		Config: map[string]any{
+			"command": []string{"python", "app.py"},
+		},
+		App: &apps.App{
+			Name:        "app_id",
+			Description: "app description",
+			Resources: []apps.AppResource{
+				{
+					Name: "job1",
+					Job: &apps.AppResourceJob{
+						Id:         "1234",
+						Permission: "CAN_MANAGE_RUN",
+					},
+				},
+				{
+					Name: "sql1",
+					SqlWarehouse: &apps.AppResourceSqlWarehouse{
+						Id:         "5678",
+						Permission: "CAN_USE",
+					},
+				},
+			},
+		},
+		Permissions: []resources.Permission{
+			{
+				Level:    "CAN_RUN",
+				UserName: "jack@gmail.com",
+			},
+			{
+				Level:                "CAN_MANAGE",
+				ServicePrincipalName: "sp",
+			},
+		},
+	}
+
+	vin, err := convert.FromTyped(src, dyn.NilValue)
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	out := schema.NewResources()
+	err = appConverter{}.Convert(ctx, "my_app", vin, out)
+	require.NoError(t, err)
+
+	app := out.App["my_app"]
+	assert.Equal(t, map[string]any{
+		"description": "app description",
+		"name":        "app_id",
+		"resources": []any{
+			map[string]any{
+				"name": "job1",
+				"job": map[string]any{
+					"id":         "1234",
+					"permission": "CAN_MANAGE_RUN",
+				},
+			},
+			map[string]any{
+				"name": "sql1",
+				"sql_warehouse": map[string]any{
+					"id":         "5678",
+					"permission": "CAN_USE",
+				},
+			},
+		},
+	}, app)
+
+	// Assert equality on the permissions
+	assert.Equal(t, &schema.ResourcePermissions{
+		AppName: "${databricks_app.my_app.name}",
+		AccessControl: []schema.ResourcePermissionsAccessControl{
+			{
+				PermissionLevel: "CAN_RUN",
+				UserName:        "jack@gmail.com",
+			},
+			{
+				PermissionLevel:      "CAN_MANAGE",
+				ServicePrincipalName: "sp",
+			},
+		},
+	}, out.Permissions["app_my_app"])
+}
+
+func TestConvertAppWithNoDescription(t *testing.T) {
+	src := resources.App{
+		SourceCodePath: "./app",
+		Config: map[string]any{
+			"command": []string{"python", "app.py"},
+		},
+		App: &apps.App{
+			Name: "app_id",
+			Resources: []apps.AppResource{
+				{
+					Name: "job1",
+					Job: &apps.AppResourceJob{
+						Id:         "1234",
+						Permission: "CAN_MANAGE_RUN",
+					},
+				},
+				{
+					Name: "sql1",
+					SqlWarehouse: &apps.AppResourceSqlWarehouse{
+						Id:         "5678",
+						Permission: "CAN_USE",
+					},
+				},
+			},
+		},
+	}
+
+	vin, err := convert.FromTyped(src, dyn.NilValue)
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	out := schema.NewResources()
+	err = appConverter{}.Convert(ctx, "my_app", vin, out)
+	require.NoError(t, err)
+
+	app := out.App["my_app"]
+	assert.Equal(t, map[string]any{
+		"name":        "app_id",
+		"description": "", // Due to Apps API always returning a description field, we set it in the output as well to avoid permanent TF drift
+		"resources": []any{
+			map[string]any{
+				"name": "job1",
+				"job": map[string]any{
+					"id":         "1234",
+					"permission": "CAN_MANAGE_RUN",
+				},
+			},
+			map[string]any{
+				"name": "sql1",
+				"sql_warehouse": map[string]any{
+					"id":         "5678",
+					"permission": "CAN_USE",
+				},
+			},
+		},
+	}, app)
+}
diff --git a/bundle/deploy/terraform/util.go b/bundle/deploy/terraform/util.go
index 4da015c23..90dfe37b2 100644
--- a/bundle/deploy/terraform/util.go
+++ b/bundle/deploy/terraform/util.go
@@ -33,7 +33,12 @@ type stateResourceInstance struct {
 }
 
 type stateInstanceAttributes struct {
-	ID   string `json:"id"`
+	ID string `json:"id"`
+
+	// Some resources such as Apps do not have an ID, so we use the name instead.
+	// We need this for cases when such resource is removed from bundle config but
+	// exists in the workspace still so we can correctly display its summary.
+	Name string `json:"name,omitempty"`
 	ETag string `json:"etag,omitempty"`
 }
 
diff --git a/bundle/deploy/terraform/util_test.go b/bundle/deploy/terraform/util_test.go
index 74b329259..5d1310392 100644
--- a/bundle/deploy/terraform/util_test.go
+++ b/bundle/deploy/terraform/util_test.go
@@ -97,7 +97,7 @@ func TestParseResourcesStateWithExistingStateFile(t *testing.T) {
 				Type: "databricks_pipeline",
 				Name: "test_pipeline",
 				Instances: []stateResourceInstance{
-					{Attributes: stateInstanceAttributes{ID: "123"}},
+					{Attributes: stateInstanceAttributes{ID: "123", Name: "test_pipeline"}},
 				},
 			},
 		},
diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml
index 5283a431b..28d29798a 100644
--- a/bundle/internal/schema/annotations.yml
+++ b/bundle/internal/schema/annotations.yml
@@ -147,6 +147,9 @@ github.com/databricks/cli/bundle/config.Python:
       If enabled, Python code will execute within this environment. If disabled,
       it defaults to using the Python interpreter available in the current shell.
 github.com/databricks/cli/bundle/config.Resources:
+  "apps":
+    "description": |-
+      PLACEHOLDER
   "clusters":
     "description": |-
       The cluster definitions for the bundle.
@@ -371,6 +374,64 @@ github.com/databricks/cli/bundle/config.Workspace:
   "state_path":
     "description": |-
       The workspace state path
+github.com/databricks/cli/bundle/config/resources.App:
+  "active_deployment":
+    "description": |-
+      PLACEHOLDER
+  "app_status":
+    "description": |-
+      PLACEHOLDER
+  "compute_status":
+    "description": |-
+      PLACEHOLDER
+  "config":
+    "description": |-
+      PLACEHOLDER
+  "create_time":
+    "description": |-
+      PLACEHOLDER
+  "creator":
+    "description": |-
+      PLACEHOLDER
+  "default_source_code_path":
+    "description": |-
+      PLACEHOLDER
+  "description":
+    "description": |-
+      PLACEHOLDER
+  "name":
+    "description": |-
+      PLACEHOLDER
+  "pending_deployment":
+    "description": |-
+      PLACEHOLDER
+  "permissions":
+    "description": |-
+      PLACEHOLDER
+  "resources":
+    "description": |-
+      PLACEHOLDER
+  "service_principal_client_id":
+    "description": |-
+      PLACEHOLDER
+  "service_principal_id":
+    "description": |-
+      PLACEHOLDER
+  "service_principal_name":
+    "description": |-
+      PLACEHOLDER
+  "source_code_path":
+    "description": |-
+      PLACEHOLDER
+  "update_time":
+    "description": |-
+      PLACEHOLDER
+  "updater":
+    "description": |-
+      PLACEHOLDER
+  "url":
+    "description": |-
+      PLACEHOLDER
 github.com/databricks/cli/bundle/config/resources.Grant:
   "principal":
     "description": |-
@@ -459,3 +520,103 @@ github.com/databricks/cli/bundle/config/variable.Variable:
   "type":
     "description": |-
       The type of the variable.
+github.com/databricks/databricks-sdk-go/service/apps.AppDeployment:
+  "create_time":
+    "description": |-
+      PLACEHOLDER
+  "creator":
+    "description": |-
+      PLACEHOLDER
+  "deployment_artifacts":
+    "description": |-
+      PLACEHOLDER
+  "deployment_id":
+    "description": |-
+      PLACEHOLDER
+  "mode":
+    "description": |-
+      PLACEHOLDER
+  "source_code_path":
+    "description": |-
+      PLACEHOLDER
+  "status":
+    "description": |-
+      PLACEHOLDER
+  "update_time":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts:
+  "source_code_path":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus:
+  "message":
+    "description": |-
+      PLACEHOLDER
+  "state":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppResource:
+  "description":
+    "description": |-
+      PLACEHOLDER
+  "job":
+    "description": |-
+      PLACEHOLDER
+  "name":
+    "description": |-
+      PLACEHOLDER
+  "secret":
+    "description": |-
+      PLACEHOLDER
+  "serving_endpoint":
+    "description": |-
+      PLACEHOLDER
+  "sql_warehouse":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob:
+  "id":
+    "description": |-
+      PLACEHOLDER
+  "permission":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret:
+  "key":
+    "description": |-
+      PLACEHOLDER
+  "permission":
+    "description": |-
+      PLACEHOLDER
+  "scope":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint:
+  "name":
+    "description": |-
+      PLACEHOLDER
+  "permission":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse:
+  "id":
+    "description": |-
+      PLACEHOLDER
+  "permission":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus:
+  "message":
+    "description": |-
+      PLACEHOLDER
+  "state":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus:
+  "message":
+    "description": |-
+      PLACEHOLDER
+  "state":
+    "description": |-
+      PLACEHOLDER
diff --git a/bundle/permissions/mutator.go b/bundle/permissions/mutator.go
index cd7cbf40c..8a0057dee 100644
--- a/bundle/permissions/mutator.go
+++ b/bundle/permissions/mutator.go
@@ -51,6 +51,10 @@ var (
 			CAN_MANAGE: "CAN_MANAGE",
 			CAN_VIEW:   "CAN_READ",
 		},
+		"apps": {
+			CAN_MANAGE: "CAN_MANAGE",
+			CAN_VIEW:   "CAN_USE",
+		},
 	}
 )
 
diff --git a/bundle/permissions/mutator_test.go b/bundle/permissions/mutator_test.go
index 15586e979..1f7897cae 100644
--- a/bundle/permissions/mutator_test.go
+++ b/bundle/permissions/mutator_test.go
@@ -58,6 +58,10 @@ func TestApplyBundlePermissions(t *testing.T) {
 					"dashboard_1": {},
 					"dashboard_2": {},
 				},
+				Apps: map[string]*resources.App{
+					"app_1": {},
+					"app_2": {},
+				},
 			},
 		},
 	}
@@ -114,6 +118,10 @@ func TestApplyBundlePermissions(t *testing.T) {
 	require.Len(t, b.Config.Resources.Dashboards["dashboard_1"].Permissions, 2)
 	require.Contains(t, b.Config.Resources.Dashboards["dashboard_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"})
 	require.Contains(t, b.Config.Resources.Dashboards["dashboard_1"].Permissions, resources.Permission{Level: "CAN_READ", GroupName: "TestGroup"})
+
+	require.Len(t, b.Config.Resources.Apps["app_1"].Permissions, 2)
+	require.Contains(t, b.Config.Resources.Apps["app_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"})
+	require.Contains(t, b.Config.Resources.Apps["app_1"].Permissions, resources.Permission{Level: "CAN_USE", GroupName: "TestGroup"})
 }
 
 func TestWarningOnOverlapPermission(t *testing.T) {
diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go
index 16595611f..c6ec04962 100644
--- a/bundle/phases/deploy.go
+++ b/bundle/phases/deploy.go
@@ -5,6 +5,7 @@ import (
 	"errors"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/apps"
 	"github.com/databricks/cli/bundle/artifacts"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/mutator"
@@ -135,6 +136,8 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator {
 		bundle.Seq(
 			terraform.StatePush(),
 			terraform.Load(),
+			apps.InterpolateVariables(),
+			apps.UploadConfig(),
 			metadata.Compute(),
 			metadata.Upload(),
 			bundle.LogString("Deployment complete!"),
diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go
index 913685bcf..50df5634a 100644
--- a/bundle/phases/initialize.go
+++ b/bundle/phases/initialize.go
@@ -2,6 +2,7 @@ package phases
 
 import (
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/apps"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/mutator"
 	pythonmutator "github.com/databricks/cli/bundle/config/mutator/python"
@@ -71,6 +72,7 @@ func Initialize() bundle.Mutator {
 			mutator.MergeJobParameters(),
 			mutator.MergeJobTasks(),
 			mutator.MergePipelineClusters(),
+			mutator.MergeApps(),
 
 			// Provide permission config errors & warnings after initializing all variables
 			permissions.PermissionDiagnostics(),
@@ -89,6 +91,8 @@ func Initialize() bundle.Mutator {
 			mutator.TranslatePaths(),
 			trampoline.WrapperWarning(),
 
+			apps.Validate(),
+
 			permissions.ValidateSharedRootPermissions(),
 			permissions.ApplyBundlePermissions(),
 			permissions.FilterCurrentUser(),
diff --git a/bundle/run/app.go b/bundle/run/app.go
new file mode 100644
index 000000000..11030beda
--- /dev/null
+++ b/bundle/run/app.go
@@ -0,0 +1,212 @@
+package run
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/run/output"
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/databricks-sdk-go/service/apps"
+	"github.com/spf13/cobra"
+)
+
+func logProgress(ctx context.Context, msg string) {
+	if msg == "" {
+		return
+	}
+	cmdio.LogString(ctx, "✓ "+msg)
+}
+
+type appRunner struct {
+	key
+
+	bundle *bundle.Bundle
+	app    *resources.App
+}
+
+func (a *appRunner) Name() string {
+	if a.app == nil {
+		return ""
+	}
+
+	return a.app.Name
+}
+
+func isAppStopped(app *apps.App) bool {
+	return app.ComputeStatus == nil ||
+		(app.ComputeStatus.State == apps.ComputeStateStopped || app.ComputeStatus.State == apps.ComputeStateError)
+}
+
+func (a *appRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, error) {
+	app := a.app
+	b := a.bundle
+	if app == nil {
+		return nil, errors.New("app is not defined")
+	}
+
+	logProgress(ctx, "Getting the status of the app "+app.Name)
+	w := b.WorkspaceClient()
+
+	// Check the status of the app first.
+	createdApp, err := w.Apps.Get(ctx, apps.GetAppRequest{Name: app.Name})
+	if err != nil {
+		return nil, err
+	}
+
+	if createdApp.AppStatus != nil {
+		logProgress(ctx, fmt.Sprintf("App is in %s state", createdApp.AppStatus.State))
+	}
+
+	if createdApp.ComputeStatus != nil {
+		logProgress(ctx, fmt.Sprintf("App compute is in %s state", createdApp.ComputeStatus.State))
+	}
+
+	// There could be 2 reasons why the app is not running:
+	// 1. The app is new and was never deployed yet.
+	// 2. The app was stopped (compute not running).
+	// We need to start the app only if the compute is not running.
+	if isAppStopped(createdApp) {
+		err := a.start(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Deploy the app.
+	err = a.deploy(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	cmdio.LogString(ctx, "You can access the app at "+createdApp.Url)
+	return nil, nil
+}
+
+func (a *appRunner) start(ctx context.Context) error {
+	app := a.app
+	b := a.bundle
+	w := b.WorkspaceClient()
+
+	logProgress(ctx, "Starting the app "+app.Name)
+	wait, err := w.Apps.Start(ctx, apps.StartAppRequest{Name: app.Name})
+	if err != nil {
+		return err
+	}
+
+	startedApp, err := wait.OnProgress(func(p *apps.App) {
+		if p.AppStatus == nil {
+			return
+		}
+		logProgress(ctx, "App is starting...")
+	}).Get()
+	if err != nil {
+		return err
+	}
+
+	// After the app is started (meaning the compute is running), the API will return the app object with the
+	// active and pending deployments fields (if any). If there are active or pending deployments,
+	// we need to wait for them to complete before we can do the new deployment.
+	// Otherwise, the new deployment will fail.
+	// Thus, we first wait for the active deployment to complete.
+	if startedApp.ActiveDeployment != nil &&
+		startedApp.ActiveDeployment.Status.State == apps.AppDeploymentStateInProgress {
+		logProgress(ctx, "Waiting for the active deployment to complete...")
+		_, err = w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, startedApp.ActiveDeployment.DeploymentId, 20*time.Minute, nil)
+		if err != nil {
+			return err
+		}
+		logProgress(ctx, "Active deployment is completed!")
+	}
+
+	// Then, we wait for the pending deployment to complete.
+	if startedApp.PendingDeployment != nil &&
+		startedApp.PendingDeployment.Status.State == apps.AppDeploymentStateInProgress {
+		logProgress(ctx, "Waiting for the pending deployment to complete...")
+		_, err = w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, startedApp.PendingDeployment.DeploymentId, 20*time.Minute, nil)
+		if err != nil {
+			return err
+		}
+		logProgress(ctx, "Pending deployment is completed!")
+	}
+
+	logProgress(ctx, "App is started!")
+	return nil
+}
+
+func (a *appRunner) deploy(ctx context.Context) error {
+	app := a.app
+	b := a.bundle
+	w := b.WorkspaceClient()
+
+	wait, err := w.Apps.Deploy(ctx, apps.CreateAppDeploymentRequest{
+		AppName: app.Name,
+		AppDeployment: &apps.AppDeployment{
+			Mode:           apps.AppDeploymentModeSnapshot,
+			SourceCodePath: app.SourceCodePath,
+		},
+	})
+	// If deploy returns an error, then there's an active deployment in progress, wait for it to complete.
+	if err != nil {
+		return err
+	}
+
+	_, err = wait.OnProgress(func(ad *apps.AppDeployment) {
+		if ad.Status == nil {
+			return
+		}
+		logProgress(ctx, ad.Status.Message)
+	}).Get()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (a *appRunner) Cancel(ctx context.Context) error {
+	// We should cancel the app by stopping it.
+	app := a.app
+	b := a.bundle
+	if app == nil {
+		return errors.New("app is not defined")
+	}
+
+	w := b.WorkspaceClient()
+
+	logProgress(ctx, "Stopping app "+app.Name)
+	wait, err := w.Apps.Stop(ctx, apps.StopAppRequest{Name: app.Name})
+	if err != nil {
+		return err
+	}
+
+	_, err = wait.OnProgress(func(p *apps.App) {
+		if p.AppStatus == nil {
+			return
+		}
+		logProgress(ctx, p.AppStatus.Message)
+	}).Get()
+
+	logProgress(ctx, "App is stopped!")
+	return err
+}
+
+func (a *appRunner) Restart(ctx context.Context, opts *Options) (output.RunOutput, error) {
+	// We should restart the app by just running it again meaning a new app deployment will be done.
+	return a.Run(ctx, opts)
+}
+
+func (a *appRunner) ParseArgs(args []string, opts *Options) error {
+	if len(args) == 0 {
+		return nil
+	}
+
+	return fmt.Errorf("received %d unexpected positional arguments", len(args))
+}
+
+func (a *appRunner) CompleteArgs(args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+	return nil, cobra.ShellCompDirectiveNoFileComp
+}
diff --git a/bundle/run/app_test.go b/bundle/run/app_test.go
new file mode 100644
index 000000000..44ff698e5
--- /dev/null
+++ b/bundle/run/app_test.go
@@ -0,0 +1,216 @@
+package run
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/internal/bundletest"
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/flags"
+	"github.com/databricks/cli/libs/vfs"
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/service/apps"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+type testAppRunner struct {
+	m   *mocks.MockWorkspaceClient
+	b   *bundle.Bundle
+	ctx context.Context
+}
+
+func (ta *testAppRunner) run(t *testing.T) {
+	r := appRunner{
+		key:    "my_app",
+		bundle: ta.b,
+		app:    ta.b.Config.Resources.Apps["my_app"],
+	}
+
+	_, err := r.Run(ta.ctx, &Options{})
+	require.NoError(t, err)
+}
+
+func setupBundle(t *testing.T) (context.Context, *bundle.Bundle, *mocks.MockWorkspaceClient) {
+	root := t.TempDir()
+	err := os.MkdirAll(filepath.Join(root, "my_app"), 0o700)
+	require.NoError(t, err)
+
+	b := &bundle.Bundle{
+		BundleRootPath: root,
+		SyncRoot:       vfs.MustNew(root),
+		Config: config.Root{
+			Workspace: config.Workspace{
+				RootPath: "/Workspace/Users/foo@bar.com/",
+			},
+			Resources: config.Resources{
+				Apps: map[string]*resources.App{
+					"my_app": {
+						App: &apps.App{
+							Name: "my_app",
+						},
+						SourceCodePath: "./my_app",
+						Config: map[string]any{
+							"command": []string{"echo", "hello"},
+							"env": []map[string]string{
+								{"name": "MY_APP", "value": "my value"},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	mwc := mocks.NewMockWorkspaceClient(t)
+	b.SetWorkpaceClient(mwc.WorkspaceClient)
+	bundletest.SetLocation(b, "resources.apps.my_app", []dyn.Location{{File: "./databricks.yml"}})
+
+	ctx := context.Background()
+	ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputText, &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}, "", "..."))
+	ctx = cmdio.NewContext(ctx, cmdio.NewLogger(flags.ModeAppend))
+
+	diags := bundle.Apply(ctx, b, bundle.Seq(
+		mutator.DefineDefaultWorkspacePaths(),
+		mutator.TranslatePaths(),
+	))
+	require.Empty(t, diags)
+
+	return ctx, b, mwc
+}
+
+func setupTestApp(t *testing.T, initialAppState apps.ApplicationState, initialComputeState apps.ComputeState) *testAppRunner {
+	ctx, b, mwc := setupBundle(t)
+
+	appApi := mwc.GetMockAppsAPI()
+	appApi.EXPECT().Get(mock.Anything, apps.GetAppRequest{
+		Name: "my_app",
+	}).Return(&apps.App{
+		Name: "my_app",
+		AppStatus: &apps.ApplicationStatus{
+			State: initialAppState,
+		},
+		ComputeStatus: &apps.ComputeStatus{
+			State: initialComputeState,
+		},
+	}, nil)
+
+	wait := &apps.WaitGetDeploymentAppSucceeded[apps.AppDeployment]{
+		Poll: func(_ time.Duration, _ func(*apps.AppDeployment)) (*apps.AppDeployment, error) {
+			return nil, nil
+		},
+	}
+	appApi.EXPECT().Deploy(mock.Anything, apps.CreateAppDeploymentRequest{
+		AppName: "my_app",
+		AppDeployment: &apps.AppDeployment{
+			Mode:           apps.AppDeploymentModeSnapshot,
+			SourceCodePath: "/Workspace/Users/foo@bar.com/files/my_app",
+		},
+	}).Return(wait, nil)
+
+	return &testAppRunner{
+		m:   mwc,
+		b:   b,
+		ctx: ctx,
+	}
+}
+
+func TestAppRunStartedApp(t *testing.T) {
+	r := setupTestApp(t, apps.ApplicationStateRunning, apps.ComputeStateActive)
+	r.run(t)
+}
+
+func TestAppRunStoppedApp(t *testing.T) {
+	r := setupTestApp(t, apps.ApplicationStateCrashed, apps.ComputeStateStopped)
+
+	appsApi := r.m.GetMockAppsAPI()
+	appsApi.EXPECT().Start(mock.Anything, apps.StartAppRequest{
+		Name: "my_app",
+	}).Return(&apps.WaitGetAppActive[apps.App]{
+		Poll: func(_ time.Duration, _ func(*apps.App)) (*apps.App, error) {
+			return &apps.App{
+				Name: "my_app",
+				AppStatus: &apps.ApplicationStatus{
+					State: apps.ApplicationStateRunning,
+				},
+				ComputeStatus: &apps.ComputeStatus{
+					State: apps.ComputeStateActive,
+				},
+			}, nil
+		},
+	}, nil)
+
+	r.run(t)
+}
+
+func TestAppRunWithAnActiveDeploymentInProgress(t *testing.T) {
+	r := setupTestApp(t, apps.ApplicationStateCrashed, apps.ComputeStateStopped)
+
+	appsApi := r.m.GetMockAppsAPI()
+	appsApi.EXPECT().Start(mock.Anything, apps.StartAppRequest{
+		Name: "my_app",
+	}).Return(&apps.WaitGetAppActive[apps.App]{
+		Poll: func(_ time.Duration, _ func(*apps.App)) (*apps.App, error) {
+			return &apps.App{
+				Name: "my_app",
+				AppStatus: &apps.ApplicationStatus{
+					State: apps.ApplicationStateRunning,
+				},
+				ComputeStatus: &apps.ComputeStatus{
+					State: apps.ComputeStateActive,
+				},
+				ActiveDeployment: &apps.AppDeployment{
+					DeploymentId: "active_deployment_id",
+					Status: &apps.AppDeploymentStatus{
+						State: apps.AppDeploymentStateInProgress,
+					},
+				},
+				PendingDeployment: &apps.AppDeployment{
+					DeploymentId: "pending_deployment_id",
+					Status: &apps.AppDeploymentStatus{
+						State: apps.AppDeploymentStateCancelled,
+					},
+				},
+			}, nil
+		},
+	}, nil)
+
+	appsApi.EXPECT().WaitGetDeploymentAppSucceeded(mock.Anything, "my_app", "active_deployment_id", mock.Anything, mock.Anything).Return(nil, nil)
+
+	r.run(t)
+}
+
+func TestStopApp(t *testing.T) {
+	ctx, b, mwc := setupBundle(t)
+	appsApi := mwc.GetMockAppsAPI()
+	appsApi.EXPECT().Stop(mock.Anything, apps.StopAppRequest{
+		Name: "my_app",
+	}).Return(&apps.WaitGetAppStopped[apps.App]{
+		Poll: func(_ time.Duration, _ func(*apps.App)) (*apps.App, error) {
+			return &apps.App{
+				Name: "my_app",
+				AppStatus: &apps.ApplicationStatus{
+					State: apps.ApplicationStateUnavailable,
+				},
+			}, nil
+		},
+	}, nil)
+
+	r := appRunner{
+		key:    "my_app",
+		bundle: b,
+		app:    b.Config.Resources.Apps["my_app"],
+	}
+
+	err := r.Cancel(ctx)
+	require.NoError(t, err)
+}
diff --git a/bundle/run/runner.go b/bundle/run/runner.go
index 4c907d068..23c2c0a41 100644
--- a/bundle/run/runner.go
+++ b/bundle/run/runner.go
@@ -42,7 +42,7 @@ type Runner interface {
 // IsRunnable returns a filter that only allows runnable resources.
 func IsRunnable(ref refs.Reference) bool {
 	switch ref.Resource.(type) {
-	case *resources.Job, *resources.Pipeline:
+	case *resources.Job, *resources.Pipeline, *resources.App:
 		return true
 	default:
 		return false
@@ -56,6 +56,12 @@ func ToRunner(b *bundle.Bundle, ref refs.Reference) (Runner, error) {
 		return &jobRunner{key: key(ref.KeyWithType), bundle: b, job: resource}, nil
 	case *resources.Pipeline:
 		return &pipelineRunner{key: key(ref.KeyWithType), bundle: b, pipeline: resource}, nil
+	case *resources.App:
+		return &appRunner{
+			key:    key(ref.KeyWithType),
+			bundle: b,
+			app:    resource,
+		}, nil
 	default:
 		return nil, fmt.Errorf("unsupported resource type: %T", resource)
 	}
diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json
index 2f78ffcca..81ae1329f 100644
--- a/bundle/schema/jsonschema.json
+++ b/bundle/schema/jsonschema.json
@@ -59,6 +59,81 @@
         "cli": {
           "bundle": {
             "config": {
+              "resources.App": {
+                "oneOf": [
+                  {
+                    "type": "object",
+                    "properties": {
+                      "active_deployment": {
+                        "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppDeployment"
+                      },
+                      "app_status": {
+                        "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus"
+                      },
+                      "compute_status": {
+                        "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus"
+                      },
+                      "config": {
+                        "$ref": "#/$defs/map/interface"
+                      },
+                      "create_time": {
+                        "$ref": "#/$defs/string"
+                      },
+                      "creator": {
+                        "$ref": "#/$defs/string"
+                      },
+                      "default_source_code_path": {
+                        "$ref": "#/$defs/string"
+                      },
+                      "description": {
+                        "$ref": "#/$defs/string"
+                      },
+                      "name": {
+                        "$ref": "#/$defs/string"
+                      },
+                      "pending_deployment": {
+                        "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppDeployment"
+                      },
+                      "permissions": {
+                        "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission"
+                      },
+                      "resources": {
+                        "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/apps.AppResource"
+                      },
+                      "service_principal_client_id": {
+                        "$ref": "#/$defs/string"
+                      },
+                      "service_principal_id": {
+                        "$ref": "#/$defs/int64"
+                      },
+                      "service_principal_name": {
+                        "$ref": "#/$defs/string"
+                      },
+                      "source_code_path": {
+                        "$ref": "#/$defs/string"
+                      },
+                      "update_time": {
+                        "$ref": "#/$defs/string"
+                      },
+                      "updater": {
+                        "$ref": "#/$defs/string"
+                      },
+                      "url": {
+                        "$ref": "#/$defs/string"
+                      }
+                    },
+                    "additionalProperties": false,
+                    "required": [
+                      "source_code_path",
+                      "name"
+                    ]
+                  },
+                  {
+                    "type": "string",
+                    "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                  }
+                ]
+              },
               "resources.Cluster": {
                 "oneOf": [
                   {
@@ -1273,6 +1348,9 @@
                 {
                   "type": "object",
                   "properties": {
+                    "apps": {
+                      "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.App"
+                    },
                     "clusters": {
                       "description": "The cluster definitions for the bundle.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Cluster",
@@ -1528,6 +1606,280 @@
         },
         "databricks-sdk-go": {
           "service": {
+            "apps.AppDeployment": {
+              "oneOf": [
+                {
+                  "type": "object",
+                  "properties": {
+                    "create_time": {
+                      "$ref": "#/$defs/string"
+                    },
+                    "creator": {
+                      "$ref": "#/$defs/string"
+                    },
+                    "deployment_artifacts": {
+                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts"
+                    },
+                    "deployment_id": {
+                      "$ref": "#/$defs/string"
+                    },
+                    "mode": {
+                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentMode"
+                    },
+                    "source_code_path": {
+                      "$ref": "#/$defs/string"
+                    },
+                    "status": {
+                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus"
+                    },
+                    "update_time": {
+                      "$ref": "#/$defs/string"
+                    }
+                  },
+                  "additionalProperties": false
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
+              ]
+            },
+            "apps.AppDeploymentArtifacts": {
+              "oneOf": [
+                {
+                  "type": "object",
+                  "properties": {
+                    "source_code_path": {
+                      "$ref": "#/$defs/string"
+                    }
+                  },
+                  "additionalProperties": false
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
+              ]
+            },
+            "apps.AppDeploymentMode": {
+              "type": "string"
+            },
+            "apps.AppDeploymentState": {
+              "type": "string"
+            },
+            "apps.AppDeploymentStatus": {
+              "oneOf": [
+                {
+                  "type": "object",
+                  "properties": {
+                    "message": {
+                      "$ref": "#/$defs/string"
+                    },
+                    "state": {
+                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentState"
+                    }
+                  },
+                  "additionalProperties": false
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
+              ]
+            },
+            "apps.AppResource": {
+              "oneOf": [
+                {
+                  "type": "object",
+                  "properties": {
+                    "description": {
+                      "$ref": "#/$defs/string"
+                    },
+                    "job": {
+                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob"
+                    },
+                    "name": {
+                      "$ref": "#/$defs/string"
+                    },
+                    "secret": {
+                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret"
+                    },
+                    "serving_endpoint": {
+                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint"
+                    },
+                    "sql_warehouse": {
+                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse"
+                    }
+                  },
+                  "additionalProperties": false,
+                  "required": [
+                    "name"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
+              ]
+            },
+            "apps.AppResourceJob": {
+              "oneOf": [
+                {
+                  "type": "object",
+                  "properties": {
+                    "id": {
+                      "$ref": "#/$defs/string"
+                    },
+                    "permission": {
+                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceJobJobPermission"
+                    }
+                  },
+                  "additionalProperties": false,
+                  "required": [
+                    "id",
+                    "permission"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
+              ]
+            },
+            "apps.AppResourceJobJobPermission": {
+              "type": "string"
+            },
+            "apps.AppResourceSecret": {
+              "oneOf": [
+                {
+                  "type": "object",
+                  "properties": {
+                    "key": {
+                      "$ref": "#/$defs/string"
+                    },
+                    "permission": {
+                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecretSecretPermission"
+                    },
+                    "scope": {
+                      "$ref": "#/$defs/string"
+                    }
+                  },
+                  "additionalProperties": false,
+                  "required": [
+                    "key",
+                    "permission",
+                    "scope"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
+              ]
+            },
+            "apps.AppResourceSecretSecretPermission": {
+              "type": "string"
+            },
+            "apps.AppResourceServingEndpoint": {
+              "oneOf": [
+                {
+                  "type": "object",
+                  "properties": {
+                    "name": {
+                      "$ref": "#/$defs/string"
+                    },
+                    "permission": {
+                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpointServingEndpointPermission"
+                    }
+                  },
+                  "additionalProperties": false,
+                  "required": [
+                    "name",
+                    "permission"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
+              ]
+            },
+            "apps.AppResourceServingEndpointServingEndpointPermission": {
+              "type": "string"
+            },
+            "apps.AppResourceSqlWarehouse": {
+              "oneOf": [
+                {
+                  "type": "object",
+                  "properties": {
+                    "id": {
+                      "$ref": "#/$defs/string"
+                    },
+                    "permission": {
+                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouseSqlWarehousePermission"
+                    }
+                  },
+                  "additionalProperties": false,
+                  "required": [
+                    "id",
+                    "permission"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
+              ]
+            },
+            "apps.AppResourceSqlWarehouseSqlWarehousePermission": {
+              "type": "string"
+            },
+            "apps.ApplicationState": {
+              "type": "string"
+            },
+            "apps.ApplicationStatus": {
+              "oneOf": [
+                {
+                  "type": "object",
+                  "properties": {
+                    "message": {
+                      "$ref": "#/$defs/string"
+                    },
+                    "state": {
+                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.ApplicationState"
+                    }
+                  },
+                  "additionalProperties": false
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
+              ]
+            },
+            "apps.ComputeState": {
+              "type": "string"
+            },
+            "apps.ComputeStatus": {
+              "oneOf": [
+                {
+                  "type": "object",
+                  "properties": {
+                    "message": {
+                      "$ref": "#/$defs/string"
+                    },
+                    "state": {
+                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.ComputeState"
+                    }
+                  },
+                  "additionalProperties": false
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
+              ]
+            },
             "catalog.MonitorCronSchedule": {
               "oneOf": [
                 {
@@ -5718,6 +6070,20 @@
           "cli": {
             "bundle": {
               "config": {
+                "resources.App": {
+                  "oneOf": [
+                    {
+                      "type": "object",
+                      "additionalProperties": {
+                        "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.App"
+                      }
+                    },
+                    {
+                      "type": "string",
+                      "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                    }
+                  ]
+                },
                 "resources.Cluster": {
                   "oneOf": [
                     {
@@ -5947,6 +6313,20 @@
           }
         }
       },
+      "interface": {
+        "oneOf": [
+          {
+            "type": "object",
+            "additionalProperties": {
+              "$ref": "#/$defs/interface"
+            }
+          },
+          {
+            "type": "string",
+            "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+          }
+        ]
+      },
       "string": {
         "oneOf": [
           {
@@ -6015,6 +6395,20 @@
           },
           "databricks-sdk-go": {
             "service": {
+              "apps.AppResource": {
+                "oneOf": [
+                  {
+                    "type": "array",
+                    "items": {
+                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResource"
+                    }
+                  },
+                  {
+                    "type": "string",
+                    "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                  }
+                ]
+              },
               "catalog.MonitorMetric": {
                 "oneOf": [
                   {
diff --git a/bundle/tests/apps/databricks.yml b/bundle/tests/apps/databricks.yml
new file mode 100644
index 000000000..ad7e93006
--- /dev/null
+++ b/bundle/tests/apps/databricks.yml
@@ -0,0 +1,71 @@
+bundle:
+  name: apps
+
+workspace:
+  host: https://acme.cloud.databricks.com/
+
+variables:
+  app_config:
+    type: complex
+    default:
+      command:
+          - "python"
+          - "app.py"
+      env:
+        - name: SOME_ENV_VARIABLE
+          value: "Some value"
+
+resources:
+  apps:
+    my_app:
+      name: "my-app"
+      description: "My App"
+      source_code_path: ./app
+      config: ${var.app_config}
+
+      resources:
+        - name: "my-sql-warehouse"
+          sql_warehouse:
+            id: 1234
+            permission: "CAN_USE"
+        - name: "my-job"
+          job:
+            id: 5678
+            permission: "CAN_MANAGE_RUN"
+      permissions:
+        - user_name: "foo@bar.com"
+          level: "CAN_VIEW"
+        - service_principal_name: "my_sp"
+          level: "CAN_MANAGE"
+
+
+targets:
+  default:
+
+  development:
+    variables:
+      app_config:
+        command:
+          - "python"
+          - "dev.py"
+        env:
+          - name: SOME_ENV_VARIABLE_2
+            value: "Some value 2"
+    resources:
+      apps:
+        my_app:
+          source_code_path: ./app-dev
+          resources:
+          - name: "my-sql-warehouse"
+            sql_warehouse:
+              id: 1234
+              permission: "CAN_MANAGE"
+          - name: "my-job"
+            job:
+              id: 5678
+              permission: "CAN_MANAGE"
+          - name: "my-secret"
+            secret:
+              key: "key"
+              scope: "scope"
+              permission: "CAN_USE"
diff --git a/bundle/tests/apps_test.go b/bundle/tests/apps_test.go
new file mode 100644
index 000000000..7fee60d14
--- /dev/null
+++ b/bundle/tests/apps_test.go
@@ -0,0 +1,60 @@
+package config_tests
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestApps(t *testing.T) {
+	b := load(t, "./apps")
+	assert.Equal(t, "apps", b.Config.Bundle.Name)
+
+	diags := bundle.Apply(context.Background(), b,
+		bundle.Seq(
+			mutator.SetVariables(),
+			mutator.ResolveVariableReferences("variables"),
+		))
+	assert.Empty(t, diags)
+
+	app := b.Config.Resources.Apps["my_app"]
+	assert.Equal(t, "my-app", app.Name)
+	assert.Equal(t, "My App", app.Description)
+	assert.Equal(t, []any{"python", "app.py"}, app.Config["command"])
+	assert.Equal(t, []any{map[string]any{"name": "SOME_ENV_VARIABLE", "value": "Some value"}}, app.Config["env"])
+
+	assert.Len(t, app.Resources, 2)
+	assert.Equal(t, "1234", app.Resources[0].SqlWarehouse.Id)
+	assert.Equal(t, "CAN_USE", string(app.Resources[0].SqlWarehouse.Permission))
+	assert.Equal(t, "5678", app.Resources[1].Job.Id)
+	assert.Equal(t, "CAN_MANAGE_RUN", string(app.Resources[1].Job.Permission))
+}
+
+func TestAppsOverride(t *testing.T) {
+	b := loadTarget(t, "./apps", "development")
+	assert.Equal(t, "apps", b.Config.Bundle.Name)
+
+	diags := bundle.Apply(context.Background(), b,
+		bundle.Seq(
+			mutator.SetVariables(),
+			mutator.ResolveVariableReferences("variables"),
+		))
+	assert.Empty(t, diags)
+	app := b.Config.Resources.Apps["my_app"]
+	assert.Equal(t, "my-app", app.Name)
+	assert.Equal(t, "My App", app.Description)
+	assert.Equal(t, []any{"python", "dev.py"}, app.Config["command"])
+	assert.Equal(t, []any{map[string]any{"name": "SOME_ENV_VARIABLE_2", "value": "Some value 2"}}, app.Config["env"])
+
+	assert.Len(t, app.Resources, 3)
+	assert.Equal(t, "1234", app.Resources[0].SqlWarehouse.Id)
+	assert.Equal(t, "CAN_MANAGE", string(app.Resources[0].SqlWarehouse.Permission))
+	assert.Equal(t, "5678", app.Resources[1].Job.Id)
+	assert.Equal(t, "CAN_MANAGE", string(app.Resources[1].Job.Permission))
+	assert.Equal(t, "key", app.Resources[2].Secret.Key)
+	assert.Equal(t, "scope", app.Resources[2].Secret.Scope)
+	assert.Equal(t, "CAN_USE", string(app.Resources[2].Secret.Permission))
+}
diff --git a/bundle/tests/loader.go b/bundle/tests/loader.go
index bb68b3059..9b246b7cc 100644
--- a/bundle/tests/loader.go
+++ b/bundle/tests/loader.go
@@ -47,6 +47,7 @@ func loadTargetWithDiags(path, env string) (*bundle.Bundle, diag.Diagnostics) {
 		mutator.MergeJobParameters(),
 		mutator.MergeJobTasks(),
 		mutator.MergePipelineClusters(),
+		mutator.MergeApps(),
 	))
 	return b, diags
 }
diff --git a/cmd/bundle/generate.go b/cmd/bundle/generate.go
index 7dea19ff9..d09c6feb4 100644
--- a/cmd/bundle/generate.go
+++ b/cmd/bundle/generate.go
@@ -17,6 +17,7 @@ func newGenerateCommand() *cobra.Command {
 	cmd.AddCommand(generate.NewGenerateJobCommand())
 	cmd.AddCommand(generate.NewGeneratePipelineCommand())
 	cmd.AddCommand(generate.NewGenerateDashboardCommand())
+	cmd.AddCommand(generate.NewGenerateAppCommand())
 	cmd.PersistentFlags().StringVar(&key, "key", "", `resource key to use for the generated configuration`)
 	return cmd
 }
diff --git a/cmd/bundle/generate/app.go b/cmd/bundle/generate/app.go
new file mode 100644
index 000000000..819b62b38
--- /dev/null
+++ b/cmd/bundle/generate/app.go
@@ -0,0 +1,166 @@
+package generate
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"path/filepath"
+
+	"github.com/databricks/cli/bundle/config/generate"
+	"github.com/databricks/cli/cmd/root"
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/yamlsaver"
+	"github.com/databricks/cli/libs/filer"
+	"github.com/databricks/cli/libs/textutil"
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/service/apps"
+	"github.com/spf13/cobra"
+
+	"gopkg.in/yaml.v3"
+)
+
+func NewGenerateAppCommand() *cobra.Command {
+	var configDir string
+	var sourceDir string
+	var appName string
+	var force bool
+
+	cmd := &cobra.Command{
+		Use:   "app",
+		Short: "Generate bundle configuration for a Databricks app",
+	}
+
+	cmd.Flags().StringVar(&appName, "existing-app-name", "", `App name to generate config for`)
+	cmd.MarkFlagRequired("existing-app-name")
+
+	cmd.Flags().StringVarP(&configDir, "config-dir", "d", filepath.Join("resources"), `Directory path where the output bundle config will be stored`)
+	cmd.Flags().StringVarP(&sourceDir, "source-dir", "s", filepath.Join("src", "app"), `Directory path where the app files will be stored`)
+	cmd.Flags().BoolVarP(&force, "force", "f", false, `Force overwrite existing files in the output directory`)
+
+	cmd.RunE = func(cmd *cobra.Command, args []string) error {
+		ctx := cmd.Context()
+		b, diags := root.MustConfigureBundle(cmd)
+		if err := diags.Error(); err != nil {
+			return diags.Error()
+		}
+
+		w := b.WorkspaceClient()
+		cmdio.LogString(ctx, fmt.Sprintf("Loading app '%s' configuration", appName))
+		app, err := w.Apps.Get(ctx, apps.GetAppRequest{Name: appName})
+		if err != nil {
+			return err
+		}
+
+		// Making sure the config directory and source directory are absolute paths.
+		if !filepath.IsAbs(configDir) {
+			configDir = filepath.Join(b.BundleRootPath, configDir)
+		}
+
+		if !filepath.IsAbs(sourceDir) {
+			sourceDir = filepath.Join(b.BundleRootPath, sourceDir)
+		}
+
+		downloader := newDownloader(w, sourceDir, configDir)
+
+		sourceCodePath := app.DefaultSourceCodePath
+		err = downloader.markDirectoryForDownload(ctx, &sourceCodePath)
+		if err != nil {
+			return err
+		}
+
+		appConfig, err := getAppConfig(ctx, app, w)
+		if err != nil {
+			return fmt.Errorf("failed to get app config: %w", err)
+		}
+
+		// Making sure the source code path is relative to the config directory.
+		rel, err := filepath.Rel(configDir, sourceDir)
+		if err != nil {
+			return err
+		}
+
+		v, err := generate.ConvertAppToValue(app, filepath.ToSlash(rel), appConfig)
+		if err != nil {
+			return err
+		}
+
+		appKey := cmd.Flag("key").Value.String()
+		if appKey == "" {
+			appKey = textutil.NormalizeString(app.Name)
+		}
+
+		result := map[string]dyn.Value{
+			"resources": dyn.V(map[string]dyn.Value{
+				"apps": dyn.V(map[string]dyn.Value{
+					appKey: v,
+				}),
+			}),
+		}
+
+		// If there are app.yaml or app.yml files in the source code path, they will be downloaded but we don't want to include them in the bundle.
+		// We include this configuration inline, so we need to remove these files.
+		for _, configFile := range []string{"app.yml", "app.yaml"} {
+			delete(downloader.files, filepath.Join(sourceDir, configFile))
+		}
+
+		err = downloader.FlushToDisk(ctx, force)
+		if err != nil {
+			return err
+		}
+
+		filename := filepath.Join(configDir, appKey+".app.yml")
+
+		saver := yamlsaver.NewSaver()
+		err = saver.SaveAsYAML(result, filename, force)
+		if err != nil {
+			return err
+		}
+
+		cmdio.LogString(ctx, "App configuration successfully saved to "+filename)
+		return nil
+	}
+
+	return cmd
+}
+
+func getAppConfig(ctx context.Context, app *apps.App, w *databricks.WorkspaceClient) (map[string]any, error) {
+	sourceCodePath := app.DefaultSourceCodePath
+
+	f, err := filer.NewWorkspaceFilesClient(w, sourceCodePath)
+	if err != nil {
+		return nil, err
+	}
+
+	// The app config is stored in app.yml or app.yaml file in the source code path.
+	configFileNames := []string{"app.yml", "app.yaml"}
+	for _, configFile := range configFileNames {
+		r, err := f.Read(ctx, configFile)
+		if err != nil {
+			if errors.Is(err, fs.ErrNotExist) {
+				continue
+			}
+			return nil, err
+		}
+		defer r.Close()
+
+		cmdio.LogString(ctx, "Reading app configuration from "+configFile)
+		content, err := io.ReadAll(r)
+		if err != nil {
+			return nil, err
+		}
+
+		var appConfig map[string]any
+		err = yaml.Unmarshal(content, &appConfig)
+		if err != nil {
+			cmdio.LogString(ctx, fmt.Sprintf("Failed to parse app configuration:\n%s\nerr: %v", string(content), err))
+			return nil, nil
+		}
+
+		return appConfig, nil
+	}
+
+	return nil, nil
+}
diff --git a/cmd/bundle/generate/utils.go b/cmd/bundle/generate/utils.go
index dbfad9438..cbea0bfcc 100644
--- a/cmd/bundle/generate/utils.go
+++ b/cmd/bundle/generate/utils.go
@@ -13,6 +13,7 @@ import (
 	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/databricks/databricks-sdk-go/service/pipelines"
+	"github.com/databricks/databricks-sdk-go/service/workspace"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -63,6 +64,37 @@ func (n *downloader) markFileForDownload(ctx context.Context, filePath *string)
 	return nil
 }
 
+func (n *downloader) markDirectoryForDownload(ctx context.Context, dirPath *string) error {
+	_, err := n.w.Workspace.GetStatusByPath(ctx, *dirPath)
+	if err != nil {
+		return err
+	}
+
+	objects, err := n.w.Workspace.RecursiveList(ctx, *dirPath)
+	if err != nil {
+		return err
+	}
+
+	for _, obj := range objects {
+		if obj.ObjectType == workspace.ObjectTypeDirectory {
+			continue
+		}
+
+		err := n.markFileForDownload(ctx, &obj.Path)
+		if err != nil {
+			return err
+		}
+	}
+
+	rel, err := filepath.Rel(n.configDir, n.sourceDir)
+	if err != nil {
+		return err
+	}
+
+	*dirPath = rel
+	return nil
+}
+
 func (n *downloader) markNotebookForDownload(ctx context.Context, notebookPath *string) error {
 	info, err := n.w.Workspace.GetStatusByPath(ctx, *notebookPath)
 	if err != nil {
diff --git a/integration/bundle/apps_test.go b/integration/bundle/apps_test.go
new file mode 100644
index 000000000..f15d8aabc
--- /dev/null
+++ b/integration/bundle/apps_test.go
@@ -0,0 +1,113 @@
+package bundle_test
+
+import (
+	"fmt"
+	"io"
+	"testing"
+
+	"github.com/databricks/cli/integration/internal/acc"
+	"github.com/databricks/cli/internal/testutil"
+	"github.com/databricks/cli/libs/env"
+	"github.com/databricks/databricks-sdk-go/service/apps"
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDeployBundleWithApp(t *testing.T) {
+	ctx, wt := acc.WorkspaceTest(t)
+
+	// TODO: should only skip app run when app can be created with no_compute option.
+	if testing.Short() {
+		t.Log("Skip the app creation and run in short mode")
+		return
+	}
+
+	if testutil.GetCloud(t) == testutil.GCP {
+		t.Skip("Skipping test for GCP cloud because /api/2.0/apps is temporarily unavailable there.")
+	}
+
+	uniqueId := uuid.New().String()
+	appId := "app-%s" + uuid.New().String()[0:8]
+	nodeTypeId := testutil.GetCloud(t).NodeTypeID()
+	instancePoolId := env.Get(ctx, "TEST_INSTANCE_POOL_ID")
+
+	root := initTestTemplate(t, ctx, "apps", map[string]any{
+		"unique_id":        uniqueId,
+		"app_id":           appId,
+		"node_type_id":     nodeTypeId,
+		"spark_version":    defaultSparkVersion,
+		"instance_pool_id": instancePoolId,
+	})
+
+	t.Cleanup(func() {
+		destroyBundle(t, ctx, root)
+		app, err := wt.W.Apps.Get(ctx, apps.GetAppRequest{Name: "test-app"})
+		if err != nil {
+			require.ErrorContains(t, err, "does not exist")
+		} else {
+			require.Contains(t, []apps.ApplicationState{apps.ApplicationStateUnavailable}, app.AppStatus.State)
+		}
+	})
+
+	deployBundle(t, ctx, root)
+
+	// App should exists after bundle deployment
+	app, err := wt.W.Apps.Get(ctx, apps.GetAppRequest{Name: appId})
+	require.NoError(t, err)
+	require.NotNil(t, app)
+
+	// Check app config
+	currentUser, err := wt.W.CurrentUser.Me(ctx)
+	require.NoError(t, err)
+
+	pathToAppYml := fmt.Sprintf("/Workspace/Users/%s/.bundle/%s/files/app/app.yml", currentUser.UserName, uniqueId)
+	reader, err := wt.W.Workspace.Download(ctx, pathToAppYml)
+	require.NoError(t, err)
+
+	data, err := io.ReadAll(reader)
+	require.NoError(t, err)
+
+	job, err := wt.W.Jobs.GetBySettingsName(ctx, "test-job-with-cluster-"+uniqueId)
+	require.NoError(t, err)
+
+	content := string(data)
+	require.Contains(t, content, fmt.Sprintf(`command:
+  - flask
+  - --app
+  - app
+  - run
+env:
+  - name: JOB_ID
+    value: "%d"`, job.JobId))
+
+	// Try to run the app
+	_, out := runResourceWithStderr(t, ctx, root, "test_app")
+	require.Contains(t, out, app.Url)
+
+	// App should be in the running state
+	app, err = wt.W.Apps.Get(ctx, apps.GetAppRequest{Name: appId})
+	require.NoError(t, err)
+	require.NotNil(t, app)
+	require.Equal(t, apps.ApplicationStateRunning, app.AppStatus.State)
+
+	// Stop the app
+	wait, err := wt.W.Apps.Stop(ctx, apps.StopAppRequest{Name: appId})
+	require.NoError(t, err)
+	app, err = wait.Get()
+	require.NoError(t, err)
+	require.NotNil(t, app)
+	require.Equal(t, apps.ApplicationStateUnavailable, app.AppStatus.State)
+
+	// Try to run the app again
+	_, out = runResourceWithStderr(t, ctx, root, "test_app")
+	require.Contains(t, out, app.Url)
+
+	// App should be in the running state
+	app, err = wt.W.Apps.Get(ctx, apps.GetAppRequest{Name: appId})
+	require.NoError(t, err)
+	require.NotNil(t, app)
+	require.Equal(t, apps.ApplicationStateRunning, app.AppStatus.State)
+
+	// Redeploy it again just to check that it can be redeployed
+	deployBundle(t, ctx, root)
+}
diff --git a/integration/bundle/bundles/apps/databricks_template_schema.json b/integration/bundle/bundles/apps/databricks_template_schema.json
new file mode 100644
index 000000000..c9faeabf3
--- /dev/null
+++ b/integration/bundle/bundles/apps/databricks_template_schema.json
@@ -0,0 +1,24 @@
+{
+    "properties": {
+        "unique_id": {
+            "type": "string",
+            "description": "Unique ID for job name"
+        },
+        "app_id": {
+            "type": "string",
+            "description": "Unique ID for app name"
+        },
+        "spark_version": {
+            "type": "string",
+            "description": "Spark version used for job cluster"
+        },
+        "node_type_id": {
+            "type": "string",
+            "description": "Node type id for job cluster"
+        },
+        "instance_pool_id": {
+            "type": "string",
+            "description": "Instance pool id for job cluster"
+        }
+    }
+}
diff --git a/integration/bundle/bundles/apps/template/app/app.py b/integration/bundle/bundles/apps/template/app/app.py
new file mode 100644
index 000000000..a60c786fe
--- /dev/null
+++ b/integration/bundle/bundles/apps/template/app/app.py
@@ -0,0 +1,15 @@
+import os
+
+from databricks.sdk import WorkspaceClient
+from flask import Flask
+
+app = Flask(__name__)
+
+
+@app.route("/")
+def home():
+    job_id = os.getenv("JOB_ID")
+
+    w = WorkspaceClient()
+    job = w.jobs.get(job_id)
+    return job.settings.name
diff --git a/integration/bundle/bundles/apps/template/databricks.yml.tmpl b/integration/bundle/bundles/apps/template/databricks.yml.tmpl
new file mode 100644
index 000000000..4d862a06f
--- /dev/null
+++ b/integration/bundle/bundles/apps/template/databricks.yml.tmpl
@@ -0,0 +1,42 @@
+bundle:
+  name: basic
+
+workspace:
+  root_path: "~/.bundle/{{.unique_id}}"
+
+resources:
+  apps:
+    test_app:
+      name: "{{.app_id}}"
+      description: "App which manages job created by this bundle"
+      source_code_path: ./app
+      config:
+        command:
+          - flask
+          - --app
+          - app
+          - run
+        env:
+          - name: JOB_ID
+            value: ${resources.jobs.foo.id}
+
+      resources:
+        - name: "app-job"
+          description: "A job for app to be able to work with"
+          job:
+            id: ${resources.jobs.foo.id}
+            permission: "CAN_MANAGE_RUN"
+
+  jobs:
+    foo:
+      name: test-job-with-cluster-{{.unique_id}}
+      tasks:
+        - task_key: my_notebook_task
+          new_cluster:
+            num_workers: 1
+            spark_version: "{{.spark_version}}"
+            node_type_id: "{{.node_type_id}}"
+            data_security_mode: USER_ISOLATION
+            instance_pool_id: "{{.instance_pool_id}}"
+          spark_python_task:
+            python_file: ./hello_world.py
diff --git a/integration/bundle/bundles/apps/template/hello_world.py b/integration/bundle/bundles/apps/template/hello_world.py
new file mode 100644
index 000000000..f301245e2
--- /dev/null
+++ b/integration/bundle/bundles/apps/template/hello_world.py
@@ -0,0 +1 @@
+print("Hello World!")
diff --git a/integration/bundle/helpers_test.go b/integration/bundle/helpers_test.go
index e884cd8c6..a537ca351 100644
--- a/integration/bundle/helpers_test.go
+++ b/integration/bundle/helpers_test.go
@@ -119,6 +119,17 @@ func runResource(t testutil.TestingT, ctx context.Context, path, key string) (st
 	return stdout.String(), err
 }
 
+func runResourceWithStderr(t testutil.TestingT, ctx context.Context, path, key string) (string, string) {
+	ctx = env.Set(ctx, "BUNDLE_ROOT", path)
+	ctx = cmdio.NewContext(ctx, cmdio.Default())
+
+	c := testcli.NewRunner(t, ctx, "bundle", "run", key)
+	stdout, stderr, err := c.Run()
+	require.NoError(t, err)
+
+	return stdout.String(), stderr.String()
+}
+
 func runResourceWithParams(t testutil.TestingT, ctx context.Context, path, key string, params ...string) (string, error) {
 	ctx = env.Set(ctx, "BUNDLE_ROOT", path)
 	ctx = cmdio.NewContext(ctx, cmdio.Default())
diff --git a/libs/dyn/merge/elements_by_key.go b/libs/dyn/merge/elements_by_key.go
index e6e640d14..df393003a 100644
--- a/libs/dyn/merge/elements_by_key.go
+++ b/libs/dyn/merge/elements_by_key.go
@@ -7,7 +7,7 @@ type elementsByKey struct {
 	keyFunc func(dyn.Value) string
 }
 
-func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
+func (e elementsByKey) doMap(_ dyn.Path, v dyn.Value, mergeFunc func(a, b dyn.Value) (dyn.Value, error)) (dyn.Value, error) {
 	// We know the type of this value is a sequence.
 	// For additional defence, return self if it is not.
 	elements, ok := v.AsSequence()
@@ -33,7 +33,7 @@ func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
 		}
 
 		// Merge this instance into the reference.
-		nv, err := Merge(ref, elements[i])
+		nv, err := mergeFunc(ref, elements[i])
 		if err != nil {
 			return v, err
 		}
@@ -55,6 +55,26 @@ func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
 	return dyn.NewValue(out, v.Locations()), nil
 }
 
+func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
+	return e.doMap(nil, v, Merge)
+}
+
+func (e elementsByKey) MapWithOverride(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+	return e.doMap(nil, v, func(a, b dyn.Value) (dyn.Value, error) {
+		return Override(a, b, OverrideVisitor{
+			VisitInsert: func(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
+				return v, nil
+			},
+			VisitDelete: func(valuePath dyn.Path, left dyn.Value) error {
+				return nil
+			},
+			VisitUpdate: func(_ dyn.Path, a, b dyn.Value) (dyn.Value, error) {
+				return b, nil
+			},
+		})
+	})
+}
+
 // ElementsByKey returns a [dyn.MapFunc] that operates on a sequence
 // where each element is a map. It groups elements by a key and merges
 // elements with the same key.
@@ -65,3 +85,7 @@ func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
 func ElementsByKey(key string, keyFunc func(dyn.Value) string) dyn.MapFunc {
 	return elementsByKey{key, keyFunc}.Map
 }
+
+func ElementsByKeyWithOverride(key string, keyFunc func(dyn.Value) string) dyn.MapFunc {
+	return elementsByKey{key, keyFunc}.MapWithOverride
+}
diff --git a/libs/dyn/merge/elements_by_key_test.go b/libs/dyn/merge/elements_by_key_test.go
index ef316cc66..09efece07 100644
--- a/libs/dyn/merge/elements_by_key_test.go
+++ b/libs/dyn/merge/elements_by_key_test.go
@@ -50,3 +50,42 @@ func TestElementByKey(t *testing.T) {
 		},
 	)
 }
+
+func TestElementByKeyWithOverride(t *testing.T) {
+	vin := dyn.V([]dyn.Value{
+		dyn.V(map[string]dyn.Value{
+			"key":   dyn.V("foo"),
+			"value": dyn.V(42),
+		}),
+		dyn.V(map[string]dyn.Value{
+			"key":   dyn.V("bar"),
+			"value": dyn.V(43),
+		}),
+		dyn.V(map[string]dyn.Value{
+			"key":        dyn.V("foo"),
+			"othervalue": dyn.V(44),
+		}),
+	})
+
+	keyFunc := func(v dyn.Value) string {
+		return strings.ToLower(v.MustString())
+	}
+
+	vout, err := dyn.MapByPath(vin, dyn.EmptyPath, ElementsByKeyWithOverride("key", keyFunc))
+	require.NoError(t, err)
+	assert.Len(t, vout.MustSequence(), 2)
+	assert.Equal(t,
+		vout.Index(0).AsAny(),
+		map[string]any{
+			"key":        "foo",
+			"othervalue": 44,
+		},
+	)
+	assert.Equal(t,
+		vout.Index(1).AsAny(),
+		map[string]any{
+			"key":   "bar",
+			"value": 43,
+		},
+	)
+}

From e1f5f60a8d5db821fcff8babf8224aef6ae0f448 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Tue, 14 Jan 2025 08:38:28 +0100
Subject: [PATCH 063/247] Filter out system clusters in cluster picker (#2131)

## Changes

As of the clusters API v2.1 the results include system clusters. On
large workspaces this can lead to long load times and include many
irrelevant results. The cluster picker should only show interactive
clusters.

Also see #1754.

## Tests

Manually confirmed the picker runs fast on a large workspace.
---
 libs/databrickscfg/cfgpickers/clusters.go      | 13 ++++++++++++-
 libs/databrickscfg/cfgpickers/clusters_test.go |  4 ++--
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/libs/databrickscfg/cfgpickers/clusters.go b/libs/databrickscfg/cfgpickers/clusters.go
index e27d13690..ba920b59b 100644
--- a/libs/databrickscfg/cfgpickers/clusters.go
+++ b/libs/databrickscfg/cfgpickers/clusters.go
@@ -136,7 +136,18 @@ func loadInteractiveClusters(ctx context.Context, w *databricks.WorkspaceClient,
 	promptSpinner := cmdio.Spinner(ctx)
 	promptSpinner <- "Loading list of clusters to select from"
 	defer close(promptSpinner)
-	all, err := w.Clusters.ListAll(ctx, compute.ListClustersRequest{})
+	all, err := w.Clusters.ListAll(ctx, compute.ListClustersRequest{
+		// Maximum page size to optimize for load time.
+		PageSize: 100,
+
+		// Filter out system clusters.
+		FilterBy: &compute.ListClustersFilterBy{
+			ClusterSources: []compute.ClusterSource{
+				compute.ClusterSourceApi,
+				compute.ClusterSourceUi,
+			},
+		},
+	})
 	if err != nil {
 		return nil, fmt.Errorf("list clusters: %w", err)
 	}
diff --git a/libs/databrickscfg/cfgpickers/clusters_test.go b/libs/databrickscfg/cfgpickers/clusters_test.go
index cde09aa44..29e190a93 100644
--- a/libs/databrickscfg/cfgpickers/clusters_test.go
+++ b/libs/databrickscfg/cfgpickers/clusters_test.go
@@ -70,7 +70,7 @@ func TestFirstCompatibleCluster(t *testing.T) {
 	cfg, server := qa.HTTPFixtures{
 		{
 			Method:   "GET",
-			Resource: "/api/2.1/clusters/list?",
+			Resource: "/api/2.1/clusters/list?filter_by.cluster_sources=API&filter_by.cluster_sources=UI&page_size=100",
 			Response: compute.ListClustersResponse{
 				Clusters: []compute.ClusterDetails{
 					{
@@ -125,7 +125,7 @@ func TestNoCompatibleClusters(t *testing.T) {
 	cfg, server := qa.HTTPFixtures{
 		{
 			Method:   "GET",
-			Resource: "/api/2.1/clusters/list?",
+			Resource: "/api/2.1/clusters/list?filter_by.cluster_sources=API&filter_by.cluster_sources=UI&page_size=100",
 			Response: compute.ListClustersResponse{
 				Clusters: []compute.ClusterDetails{
 					{

From e682eeba807bcd5654198d10ab94dde37d6976c7 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 14 Jan 2025 08:39:34 +0100
Subject: [PATCH 064/247] Pin all github actions to commit hash (#2129)

## Changes
- Pin all github actions to commit hash.
- Modify vedantmgoyal2009/winget-releaser to use tag format that
dependabot can understand.

Pinning is done by
https://github.com/databricks/cli/blob/denik/pin-actions-script/pin_actions.py
(100% chatgpt authored). Commits and tags are verified manually.

This format should be recognized by dependabot enabled in
https://github.com/databricks/cli/pull/2112

## Tests
Existing tests.
---
 .github/workflows/close-stale-issues.yml |  2 +-
 .github/workflows/external-message.yml   |  2 +-
 .github/workflows/integration-main.yml   |  2 +-
 .github/workflows/integration-pr.yml     |  2 +-
 .github/workflows/publish-winget.yml     |  2 +-
 .github/workflows/push.yml               | 18 +++++++++---------
 .github/workflows/release-snapshot.yml   | 14 +++++++-------
 .github/workflows/release.yml            | 16 ++++++++--------
 8 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/.github/workflows/close-stale-issues.yml b/.github/workflows/close-stale-issues.yml
index 7bf754319..ea9558caf 100644
--- a/.github/workflows/close-stale-issues.yml
+++ b/.github/workflows/close-stale-issues.yml
@@ -18,7 +18,7 @@ jobs:
       pull-requests: write
 
     steps:
-      - uses: actions/stale@v9
+      - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0
         with:
           stale-issue-message: This issue has not received a response in a while. If you want to keep this issue open, please leave a comment below and auto-close will be canceled.
           stale-pr-message: This PR has not received an update in a while. If you want to keep this PR open, please leave a comment below or push a new commit and auto-close will be canceled.
diff --git a/.github/workflows/external-message.yml b/.github/workflows/external-message.yml
index f06d81a47..108ca9162 100644
--- a/.github/workflows/external-message.yml
+++ b/.github/workflows/external-message.yml
@@ -25,7 +25,7 @@ jobs:
     if: "${{ github.event.pull_request.head.repo.fork }}"
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Delete old comments
         env:
diff --git a/.github/workflows/integration-main.yml b/.github/workflows/integration-main.yml
index 0b6032d50..84dd7263a 100644
--- a/.github/workflows/integration-main.yml
+++ b/.github/workflows/integration-main.yml
@@ -20,7 +20,7 @@ jobs:
     steps:
       - name: Generate GitHub App Token
         id: generate-token
-        uses: actions/create-github-app-token@v1
+        uses: actions/create-github-app-token@c1a285145b9d317df6ced56c09f525b5c2b6f755 # v1.11.1
         with:
           app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}
           private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }}
diff --git a/.github/workflows/integration-pr.yml b/.github/workflows/integration-pr.yml
index 0f9c4797a..7a62113cd 100644
--- a/.github/workflows/integration-pr.yml
+++ b/.github/workflows/integration-pr.yml
@@ -23,7 +23,7 @@ jobs:
     steps:
       - name: Generate GitHub App Token
         id: generate-token
-        uses: actions/create-github-app-token@v1
+        uses: actions/create-github-app-token@c1a285145b9d317df6ced56c09f525b5c2b6f755 # v1.11.1
         with:
           app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}
           private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }}
diff --git a/.github/workflows/publish-winget.yml b/.github/workflows/publish-winget.yml
index 267077102..eb9a72eda 100644
--- a/.github/workflows/publish-winget.yml
+++ b/.github/workflows/publish-winget.yml
@@ -16,7 +16,7 @@ jobs:
     environment: release
 
     steps:
-      - uses: vedantmgoyal2009/winget-releaser@93fd8b606a1672ec3e5c6c3bb19426be68d1a8b0 # https://github.com/vedantmgoyal2009/winget-releaser/releases/tag/v2
+      - uses: vedantmgoyal2009/winget-releaser@93fd8b606a1672ec3e5c6c3bb19426be68d1a8b0 # v2
         with:
           identifier: Databricks.DatabricksCLI
           installers-regex: 'windows_.*-signed\.zip$' # Only signed Windows releases
diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
index ddb2fb002..d998224a4 100644
--- a/.github/workflows/push.yml
+++ b/.github/workflows/push.yml
@@ -45,20 +45,20 @@ jobs:
 
     steps:
       - name: Checkout repository and submodules
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Setup Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
           go-version: 1.23.4
 
       - name: Setup Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
           python-version: '3.9'
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@887a942a15af3a7626099df99e897a18d9e5ab3a # v5.1.0
 
       - name: Set go env
         run: |
@@ -79,8 +79,8 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v5
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
           go-version: 1.23.4
           # Use different schema from regular job, to avoid overwriting the same key
@@ -95,7 +95,7 @@ jobs:
           # Exit with status code 1 if there are differences (i.e. unformatted files)
           git diff --exit-code
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v6
+        uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1
         with:
           version: v1.63.4
           args: --timeout=15m
@@ -106,10 +106,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Setup Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
           go-version: 1.23.4
           # Use different schema from regular job, to avoid overwriting the same key
diff --git a/.github/workflows/release-snapshot.yml b/.github/workflows/release-snapshot.yml
index 5c56a294e..548d93e90 100644
--- a/.github/workflows/release-snapshot.yml
+++ b/.github/workflows/release-snapshot.yml
@@ -26,13 +26,13 @@ jobs:
 
     steps:
       - name: Checkout repository and submodules
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 0
           fetch-tags: true
 
       - name: Setup Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
           go-version: 1.23.4
 
@@ -48,27 +48,27 @@ jobs:
 
       - name: Run GoReleaser
         id: releaser
-        uses: goreleaser/goreleaser-action@v6
+        uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6.1.0
         with:
           version: ~> v2
           args: release --snapshot --skip docker
 
       - name: Upload macOS binaries
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
           name: cli_darwin_snapshot
           path: |
             dist/*_darwin_*/
 
       - name: Upload Linux binaries
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
           name: cli_linux_snapshot
           path: |
             dist/*_linux_*/
 
       - name: Upload Windows binaries
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
           name: cli_windows_snapshot
           path: |
@@ -88,7 +88,7 @@ jobs:
         # Snapshot release may only be updated for commits to the main branch.
         if: github.ref == 'refs/heads/main'
 
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v1
         with:
           name: Snapshot
           prerelease: true
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 061688506..5d5811b19 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -18,13 +18,13 @@ jobs:
 
     steps:
       - name: Checkout repository and submodules
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 0
           fetch-tags: true
 
       - name: Setup Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
           go-version: 1.23.4
 
@@ -37,7 +37,7 @@ jobs:
 
       # Log into the GitHub Container Registry. The goreleaser action will create
       # the docker images and push them to the GitHub Container Registry.
-      - uses: "docker/login-action@v3"
+      - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           registry: "ghcr.io"
           username: "${{ github.actor }}"
@@ -46,11 +46,11 @@ jobs:
       # QEMU is required to build cross platform docker images using buildx.
       # It allows virtualization of the CPU architecture at the application level.
       - name: Set up QEMU dependency
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@53851d14592bedcffcf25ea515637cff71ef929a # v3.3.0
 
       - name: Run GoReleaser
         id: releaser
-        uses: goreleaser/goreleaser-action@v6
+        uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6.1.0
         with:
           version: ~> v2
           args: release
@@ -71,7 +71,7 @@ jobs:
           echo "VERSION=${VERSION:1}" >> $GITHUB_ENV
 
       - name: Update setup-cli
-        uses: actions/github-script@v7
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           github-token: ${{ secrets.DECO_GITHUB_TOKEN }}
           script: |
@@ -99,7 +99,7 @@ jobs:
           echo "VERSION=${VERSION:1}" >> $GITHUB_ENV
 
       - name: Update homebrew-tap
-        uses: actions/github-script@v7
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           github-token: ${{ secrets.DECO_GITHUB_TOKEN }}
           script: |
@@ -140,7 +140,7 @@ jobs:
           echo "VERSION=${VERSION:1}" >> $GITHUB_ENV
 
       - name: Update CLI version in the VSCode extension
-        uses: actions/github-script@v7
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           github-token: ${{ secrets.DECO_GITHUB_TOKEN }}
           script: |

From 5d9bc3b553ef8f635a562c776b556db58db500e7 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Tue, 14 Jan 2025 09:34:55 +0100
Subject: [PATCH 065/247] Allow artifact path to be located outside the sync
 root (#2128)

## Changes

We perform a check during path translation that the path being
referenced is contained in the bundle's sync root. If it isn't, it's not
a valid remote reference. However, this doesn't apply to paths that are
_always_ local, such as the artifact path. An artifact's build command
is executed in its path. Files created by the artifact build (e.g.
wheels or JARs) don't need to be in the sync root because they have a
dedicated and different upload path into `${workspace.artifact_path}`.

Therefore, this check that a path is contained in the bundle's sync root
doesn't apply to artifact paths. This change modifies the structure of
path translation to allow opting out of this check.

Fixes #1927.

## Tests

* Existing and new tests pass.
* Manually confirmed that building and using a wheel built outside the
sync root path works as expected.
* No acceptance tests because we don't run build as part of validate.
---
 bundle/config/mutator/translate_paths.go      | 205 ++++++++++++------
 bundle/config/mutator/translate_paths_apps.go |   9 +-
 .../mutator/translate_paths_artifacts.go      |  15 +-
 .../mutator/translate_paths_artifacts_test.go |  83 +++++++
 .../mutator/translate_paths_dashboards.go     |   9 +-
 bundle/config/mutator/translate_paths_jobs.go |  41 +++-
 .../mutator/translate_paths_pipelines.go      |  27 ++-
 .../artifact_a/.gitkeep                       |   0
 .../subfolder/artifact_b/.gitkeep             |   0
 .../tests/relative_path_with_includes_test.go |   4 +-
 10 files changed, 299 insertions(+), 94 deletions(-)
 create mode 100644 bundle/config/mutator/translate_paths_artifacts_test.go
 create mode 100644 bundle/tests/relative_path_with_includes/artifact_a/.gitkeep
 create mode 100644 bundle/tests/relative_path_with_includes/subfolder/artifact_b/.gitkeep

diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go
index 1915cf36e..a2c830be3 100644
--- a/bundle/config/mutator/translate_paths.go
+++ b/bundle/config/mutator/translate_paths.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/fs"
 	"net/url"
+	"os"
 	"path"
 	"path/filepath"
 	"strings"
@@ -17,6 +18,47 @@ import (
 	"github.com/databricks/cli/libs/notebook"
 )
 
+// TranslateMode specifies how a path should be translated.
+type TranslateMode int
+
+const (
+	// TranslateModeNotebook translates a path to a remote notebook.
+	TranslateModeNotebook TranslateMode = iota
+
+	// TranslateModeFile translates a path to a remote regular file.
+	TranslateModeFile
+
+	// TranslateModeDirectory translates a path to a remote directory.
+	TranslateModeDirectory
+
+	// TranslateModeLocalAbsoluteFile translates a path to the local absolute file path.
+	// It returns an error if the path does not exist or is a directory.
+	TranslateModeLocalAbsoluteFile
+
+	// TranslateModeLocalAbsoluteDirectory translates a path to the local absolute directory path.
+	// It returns an error if the path does not exist or is not a directory.
+	TranslateModeLocalAbsoluteDirectory
+
+	// TranslateModeLocalRelative translates a path to be relative to the bundle sync root path.
+	// It does not check if the path exists, nor care if it is a file or directory.
+	TranslateModeLocalRelative
+
+	// TranslateModeLocalRelativeWithPrefix translates a path to be relative to the bundle sync root path.
+	// It a "./" prefix to the path if it does not already have one.
+	// This allows for disambiguating between paths and PyPI package names.
+	TranslateModeLocalRelativeWithPrefix
+)
+
+// translateOptions control path translation behavior.
+type translateOptions struct {
+	// Mode specifies how the path should be translated.
+	Mode TranslateMode
+
+	// AllowPathOutsideSyncRoot can be set for paths that are not tied to the sync root path.
+	// This is the case for artifact paths, for example.
+	AllowPathOutsideSyncRoot bool
+}
+
 type ErrIsNotebook struct {
 	path string
 }
@@ -44,8 +86,6 @@ func (m *translatePaths) Name() string {
 	return "TranslatePaths"
 }
 
-type rewriteFunc func(literal, localFullPath, localRelPath, remotePath string) (string, error)
-
 // translateContext is a context for rewriting paths in a config.
 // It is freshly instantiated on every mutator apply call.
 // It provides access to the underlying bundle object such that
@@ -56,74 +96,90 @@ type translateContext struct {
 	// seen is a map of local paths to their corresponding remote paths.
 	// If a local path has already been successfully resolved, we do not need to resolve it again.
 	seen map[string]string
+
+	// remoteRoot is the root path of the remote workspace.
+	// It is equal to ${workspace.file_path} for regular deployments.
+	// It points to the source root path for source-linked deployments.
+	remoteRoot string
 }
 
 // rewritePath converts a given relative path from the loaded config to a new path based on the passed rewriting function
 //
 // It takes these arguments:
-//   - The argument `dir` is the directory relative to which the given relative path is.
-//   - The given relative path is both passed and written back through `*p`.
-//   - The argument `fn` is a function that performs the actual rewriting logic.
-//     This logic is different between regular files or notebooks.
+//   - The context in which the function is called.
+//   - The argument `dir` is the directory relative to which the relative path should be interpreted.
+//   - The argument `input` is the relative path to rewrite.
+//   - The argument `opts` is a struct that specifies how the path should be rewritten.
+//     It contains a `Mode` field that specifies how the path should be rewritten.
 //
-// The function returns an error if it is impossible to rewrite the given relative path.
+// The function returns the rewritten path if successful, or an error if the path could not be rewritten.
+// The returned path is an empty string if the path was not rewritten.
 func (t *translateContext) rewritePath(
+	ctx context.Context,
 	dir string,
-	p *string,
-	fn rewriteFunc,
-) error {
+	input string,
+	opts translateOptions,
+) (string, error) {
 	// We assume absolute paths point to a location in the workspace
-	if path.IsAbs(*p) {
-		return nil
+	if path.IsAbs(input) {
+		return "", nil
 	}
 
-	url, err := url.Parse(*p)
+	url, err := url.Parse(input)
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	// If the file path has scheme, it's a full path and we don't need to transform it
 	if url.Scheme != "" {
-		return nil
+		return "", nil
 	}
 
 	// Local path is relative to the directory the resource was defined in.
-	localPath := filepath.Join(dir, filepath.FromSlash(*p))
+	localPath := filepath.Join(dir, filepath.FromSlash(input))
 	if interp, ok := t.seen[localPath]; ok {
-		*p = interp
-		return nil
+		return interp, nil
 	}
 
 	// Local path must be contained in the sync root.
 	// If it isn't, it won't be synchronized into the workspace.
 	localRelPath, err := filepath.Rel(t.b.SyncRootPath, localPath)
 	if err != nil {
-		return err
+		return "", err
 	}
-	if strings.HasPrefix(localRelPath, "..") {
-		return fmt.Errorf("path %s is not contained in sync root path", localPath)
+	if !opts.AllowPathOutsideSyncRoot && !filepath.IsLocal(localRelPath) {
+		return "", fmt.Errorf("path %s is not contained in sync root path", localPath)
 	}
 
-	var workspacePath string
-	if config.IsExplicitlyEnabled(t.b.Config.Presets.SourceLinkedDeployment) {
-		workspacePath = t.b.SyncRootPath
-	} else {
-		workspacePath = t.b.Config.Workspace.FilePath
-	}
-	remotePath := path.Join(workspacePath, filepath.ToSlash(localRelPath))
-
 	// Convert local path into workspace path via specified function.
-	interp, err := fn(*p, localPath, localRelPath, remotePath)
+	var interp string
+	switch opts.Mode {
+	case TranslateModeNotebook:
+		interp, err = t.translateNotebookPath(ctx, input, localPath, localRelPath)
+	case TranslateModeFile:
+		interp, err = t.translateFilePath(ctx, input, localPath, localRelPath)
+	case TranslateModeDirectory:
+		interp, err = t.translateDirectoryPath(ctx, input, localPath, localRelPath)
+	case TranslateModeLocalAbsoluteFile:
+		interp, err = t.translateLocalAbsoluteFilePath(ctx, input, localPath, localRelPath)
+	case TranslateModeLocalAbsoluteDirectory:
+		interp, err = t.translateLocalAbsoluteDirectoryPath(ctx, input, localPath, localRelPath)
+	case TranslateModeLocalRelative:
+		interp, err = t.translateLocalRelativePath(ctx, input, localPath, localRelPath)
+	case TranslateModeLocalRelativeWithPrefix:
+		interp, err = t.translateLocalRelativeWithPrefixPath(ctx, input, localPath, localRelPath)
+	default:
+		return "", fmt.Errorf("unsupported translate mode: %d", opts.Mode)
+	}
 	if err != nil {
-		return err
+		return "", err
 	}
 
-	*p = interp
 	t.seen[localPath] = interp
-	return nil
+	return interp, nil
 }
 
-func (t *translateContext) translateNotebookPath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
+func (t *translateContext) translateNotebookPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) {
 	nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, filepath.ToSlash(localRelPath))
 	if errors.Is(err, fs.ErrNotExist) {
 		if filepath.Ext(localFullPath) != notebook.ExtensionNone {
@@ -162,10 +218,11 @@ to contain one of the following file extensions: [%s]`, literal, strings.Join(ex
 	}
 
 	// Upon import, notebooks are stripped of their extension.
-	return strings.TrimSuffix(remotePath, filepath.Ext(localFullPath)), nil
+	localRelPathNoExt := strings.TrimSuffix(localRelPath, filepath.Ext(localRelPath))
+	return path.Join(t.remoteRoot, filepath.ToSlash(localRelPathNoExt)), nil
 }
 
-func (t *translateContext) translateFilePath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
+func (t *translateContext) translateFilePath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) {
 	nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, filepath.ToSlash(localRelPath))
 	if errors.Is(err, fs.ErrNotExist) {
 		return "", fmt.Errorf("file %s not found", literal)
@@ -176,10 +233,10 @@ func (t *translateContext) translateFilePath(literal, localFullPath, localRelPat
 	if nb {
 		return "", ErrIsNotebook{localFullPath}
 	}
-	return remotePath, nil
+	return path.Join(t.remoteRoot, filepath.ToSlash(localRelPath)), nil
 }
 
-func (t *translateContext) translateDirectoryPath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
+func (t *translateContext) translateDirectoryPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) {
 	info, err := t.b.SyncRoot.Stat(filepath.ToSlash(localRelPath))
 	if err != nil {
 		return "", err
@@ -187,14 +244,10 @@ func (t *translateContext) translateDirectoryPath(literal, localFullPath, localR
 	if !info.IsDir() {
 		return "", fmt.Errorf("%s is not a directory", localFullPath)
 	}
-	return remotePath, nil
+	return path.Join(t.remoteRoot, filepath.ToSlash(localRelPath)), nil
 }
 
-func (t *translateContext) translateNoOp(literal, localFullPath, localRelPath, remotePath string) (string, error) {
-	return localRelPath, nil
-}
-
-func (t *translateContext) retainLocalAbsoluteFilePath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
+func (t *translateContext) translateLocalAbsoluteFilePath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) {
 	info, err := t.b.SyncRoot.Stat(filepath.ToSlash(localRelPath))
 	if errors.Is(err, fs.ErrNotExist) {
 		return "", fmt.Errorf("file %s not found", literal)
@@ -208,16 +261,33 @@ func (t *translateContext) retainLocalAbsoluteFilePath(literal, localFullPath, l
 	return localFullPath, nil
 }
 
-func (t *translateContext) translateNoOpWithPrefix(literal, localFullPath, localRelPath, remotePath string) (string, error) {
+func (t *translateContext) translateLocalAbsoluteDirectoryPath(ctx context.Context, literal, localFullPath, _ string) (string, error) {
+	info, err := os.Stat(localFullPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return "", fmt.Errorf("directory %s not found", literal)
+	}
+	if err != nil {
+		return "", fmt.Errorf("unable to determine if %s is a directory: %w", localFullPath, err)
+	}
+	if !info.IsDir() {
+		return "", fmt.Errorf("expected %s to be a directory but found a file", literal)
+	}
+	return localFullPath, nil
+}
+
+func (t *translateContext) translateLocalRelativePath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) {
+	return localRelPath, nil
+}
+
+func (t *translateContext) translateLocalRelativeWithPrefixPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) {
 	if !strings.HasPrefix(localRelPath, ".") {
 		localRelPath = "." + string(filepath.Separator) + localRelPath
 	}
 	return localRelPath, nil
 }
 
-func (t *translateContext) rewriteValue(p dyn.Path, v dyn.Value, fn rewriteFunc, dir string) (dyn.Value, error) {
-	out := v.MustString()
-	err := t.rewritePath(dir, &out, fn)
+func (t *translateContext) rewriteValue(ctx context.Context, p dyn.Path, v dyn.Value, dir string, opts translateOptions) (dyn.Value, error) {
+	out, err := t.rewritePath(ctx, dir, v.MustString(), opts)
 	if err != nil {
 		if target := (&ErrIsNotebook{}); errors.As(err, target) {
 			return dyn.InvalidValue, fmt.Errorf(`expected a file for "%s" but got a notebook: %w`, p, target)
@@ -228,43 +298,38 @@ func (t *translateContext) rewriteValue(p dyn.Path, v dyn.Value, fn rewriteFunc,
 		return dyn.InvalidValue, err
 	}
 
+	// If the path was not rewritten, return the original value.
+	if out == "" {
+		return v, nil
+	}
+
 	return dyn.NewValue(out, v.Locations()), nil
 }
 
-func (t *translateContext) rewriteRelativeTo(p dyn.Path, v dyn.Value, fn rewriteFunc, dir, fallback string) (dyn.Value, error) {
-	nv, err := t.rewriteValue(p, v, fn, dir)
-	if err == nil {
-		return nv, nil
-	}
-
-	// If we failed to rewrite the path, try to rewrite it relative to the fallback directory.
-	if fallback != "" {
-		nv, nerr := t.rewriteValue(p, v, fn, fallback)
-		if nerr == nil {
-			// TODO: Emit a warning that this path should be rewritten.
-			return nv, nil
-		}
-	}
-
-	return dyn.InvalidValue, err
-}
-
-func (m *translatePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
+func (m *translatePaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	t := &translateContext{
 		b:    b,
 		seen: make(map[string]string),
 	}
 
+	// Set the remote root to the sync root if source-linked deployment is enabled.
+	// Otherwise, set it to the workspace file path.
+	if config.IsExplicitlyEnabled(t.b.Config.Presets.SourceLinkedDeployment) {
+		t.remoteRoot = t.b.SyncRootPath
+	} else {
+		t.remoteRoot = t.b.Config.Workspace.FilePath
+	}
+
 	err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
 		var err error
-		for _, fn := range []func(dyn.Value) (dyn.Value, error){
+		for _, fn := range []func(context.Context, dyn.Value) (dyn.Value, error){
 			t.applyJobTranslations,
 			t.applyPipelineTranslations,
 			t.applyArtifactTranslations,
 			t.applyDashboardTranslations,
 			t.applyAppsTranslations,
 		} {
-			v, err = fn(v)
+			v, err = fn(ctx, v)
 			if err != nil {
 				return dyn.InvalidValue, err
 			}
@@ -275,6 +340,8 @@ func (m *translatePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos
 	return diag.FromErr(err)
 }
 
+// gatherFallbackPaths collects the fallback paths for relative paths in the configuration.
+// Read more about the motivation for this functionality in the "fallback" path translation tests.
 func gatherFallbackPaths(v dyn.Value, typ string) (map[string]string, error) {
 	fallback := make(map[string]string)
 	pattern := dyn.NewPattern(dyn.Key("resources"), dyn.Key(typ), dyn.AnyKey())
diff --git a/bundle/config/mutator/translate_paths_apps.go b/bundle/config/mutator/translate_paths_apps.go
index 0ed7e1928..6117ee43f 100644
--- a/bundle/config/mutator/translate_paths_apps.go
+++ b/bundle/config/mutator/translate_paths_apps.go
@@ -1,12 +1,13 @@
 package mutator
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/databricks/cli/libs/dyn"
 )
 
-func (t *translateContext) applyAppsTranslations(v dyn.Value) (dyn.Value, error) {
+func (t *translateContext) applyAppsTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) {
 	// Convert the `source_code_path` field to a remote absolute path.
 	// We use this path for app deployment to point to the source code.
 	pattern := dyn.NewPattern(
@@ -16,6 +17,10 @@ func (t *translateContext) applyAppsTranslations(v dyn.Value) (dyn.Value, error)
 		dyn.Key("source_code_path"),
 	)
 
+	opts := translateOptions{
+		Mode: TranslateModeDirectory,
+	}
+
 	return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
 		key := p[2].Key()
 		dir, err := v.Location().Directory()
@@ -23,6 +28,6 @@ func (t *translateContext) applyAppsTranslations(v dyn.Value) (dyn.Value, error)
 			return dyn.InvalidValue, fmt.Errorf("unable to determine directory for app %s: %w", key, err)
 		}
 
-		return t.rewriteRelativeTo(p, v, t.translateDirectoryPath, dir, "")
+		return t.rewriteValue(ctx, p, v, dir, opts)
 	})
 }
diff --git a/bundle/config/mutator/translate_paths_artifacts.go b/bundle/config/mutator/translate_paths_artifacts.go
index 921c00c73..8e864073f 100644
--- a/bundle/config/mutator/translate_paths_artifacts.go
+++ b/bundle/config/mutator/translate_paths_artifacts.go
@@ -1,6 +1,7 @@
 package mutator
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/databricks/cli/libs/dyn"
@@ -8,7 +9,7 @@ import (
 
 type artifactRewritePattern struct {
 	pattern dyn.Pattern
-	fn      rewriteFunc
+	opts    translateOptions
 }
 
 func (t *translateContext) artifactRewritePatterns() []artifactRewritePattern {
@@ -22,12 +23,18 @@ func (t *translateContext) artifactRewritePatterns() []artifactRewritePattern {
 	return []artifactRewritePattern{
 		{
 			base.Append(dyn.Key("path")),
-			t.translateNoOp,
+			translateOptions{
+				Mode: TranslateModeLocalAbsoluteDirectory,
+
+				// Artifact paths may be outside the sync root.
+				// They are the working directory for artifact builds.
+				AllowPathOutsideSyncRoot: true,
+			},
 		},
 	}
 }
 
-func (t *translateContext) applyArtifactTranslations(v dyn.Value) (dyn.Value, error) {
+func (t *translateContext) applyArtifactTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) {
 	var err error
 
 	for _, rewritePattern := range t.artifactRewritePatterns() {
@@ -38,7 +45,7 @@ func (t *translateContext) applyArtifactTranslations(v dyn.Value) (dyn.Value, er
 				return dyn.InvalidValue, fmt.Errorf("unable to determine directory for artifact %s: %w", key, err)
 			}
 
-			return t.rewriteRelativeTo(p, v, rewritePattern.fn, dir, "")
+			return t.rewriteValue(ctx, p, v, dir, rewritePattern.opts)
 		})
 		if err != nil {
 			return dyn.InvalidValue, err
diff --git a/bundle/config/mutator/translate_paths_artifacts_test.go b/bundle/config/mutator/translate_paths_artifacts_test.go
new file mode 100644
index 000000000..fb402b488
--- /dev/null
+++ b/bundle/config/mutator/translate_paths_artifacts_test.go
@@ -0,0 +1,83 @@
+package mutator_test
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/cli/bundle/internal/bundletest"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/vfs"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTranslatePathsArtifacts_InsideSyncRoot(t *testing.T) {
+	tmp := t.TempDir()
+	dir := filepath.Join(tmp, "bundle")
+	lib := filepath.Join(dir, "my_lib")
+	_ = os.MkdirAll(lib, 0o755)
+	_ = os.MkdirAll(dir, 0o755)
+
+	b := &bundle.Bundle{
+		SyncRootPath: dir,
+		SyncRoot:     vfs.MustNew(dir),
+		Config: config.Root{
+			Artifacts: map[string]*config.Artifact{
+				"my_artifact": {
+					Type: "wheel",
+
+					// Assume this is defined in a subdir to the sync root.
+					Path: "../my_lib",
+				},
+			},
+		},
+	}
+
+	bundletest.SetLocation(b, "artifacts", []dyn.Location{{
+		File: filepath.Join(dir, "config/artifacts.yml"),
+	}})
+
+	diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
+	require.NoError(t, diags.Error())
+
+	// Assert that the artifact path has been converted to a local absolute path.
+	assert.Equal(t, lib, b.Config.Artifacts["my_artifact"].Path)
+}
+
+func TestTranslatePathsArtifacts_OutsideSyncRoot(t *testing.T) {
+	tmp := t.TempDir()
+	lib := filepath.Join(tmp, "my_lib")
+	dir := filepath.Join(tmp, "bundle")
+	_ = os.MkdirAll(lib, 0o755)
+	_ = os.MkdirAll(dir, 0o755)
+
+	b := &bundle.Bundle{
+		SyncRootPath: dir,
+		SyncRoot:     vfs.MustNew(dir),
+		Config: config.Root{
+			Artifacts: map[string]*config.Artifact{
+				"my_artifact": {
+					Type: "wheel",
+
+					// Assume this is defined in a subdir of the bundle root.
+					Path: "../../my_lib",
+				},
+			},
+		},
+	}
+
+	bundletest.SetLocation(b, "artifacts", []dyn.Location{{
+		File: filepath.Join(dir, "config/artifacts.yml"),
+	}})
+
+	diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
+	require.NoError(t, diags.Error())
+
+	// Assert that the artifact path has been converted to a local absolute path.
+	assert.Equal(t, lib, b.Config.Artifacts["my_artifact"].Path)
+}
diff --git a/bundle/config/mutator/translate_paths_dashboards.go b/bundle/config/mutator/translate_paths_dashboards.go
index 93822a599..18c4c12e2 100644
--- a/bundle/config/mutator/translate_paths_dashboards.go
+++ b/bundle/config/mutator/translate_paths_dashboards.go
@@ -1,12 +1,13 @@
 package mutator
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/databricks/cli/libs/dyn"
 )
 
-func (t *translateContext) applyDashboardTranslations(v dyn.Value) (dyn.Value, error) {
+func (t *translateContext) applyDashboardTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) {
 	// Convert the `file_path` field to a local absolute path.
 	// We load the file at this path and use its contents for the dashboard contents.
 	pattern := dyn.NewPattern(
@@ -16,6 +17,10 @@ func (t *translateContext) applyDashboardTranslations(v dyn.Value) (dyn.Value, e
 		dyn.Key("file_path"),
 	)
 
+	opts := translateOptions{
+		Mode: TranslateModeLocalAbsoluteFile,
+	}
+
 	return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
 		key := p[2].Key()
 		dir, err := v.Location().Directory()
@@ -23,6 +28,6 @@ func (t *translateContext) applyDashboardTranslations(v dyn.Value) (dyn.Value, e
 			return dyn.InvalidValue, fmt.Errorf("unable to determine directory for dashboard %s: %w", key, err)
 		}
 
-		return t.rewriteRelativeTo(p, v, t.retainLocalAbsoluteFilePath, dir, "")
+		return t.rewriteValue(ctx, p, v, dir, opts)
 	})
 }
diff --git a/bundle/config/mutator/translate_paths_jobs.go b/bundle/config/mutator/translate_paths_jobs.go
index c29ff0ea9..148ed4466 100644
--- a/bundle/config/mutator/translate_paths_jobs.go
+++ b/bundle/config/mutator/translate_paths_jobs.go
@@ -1,6 +1,7 @@
 package mutator
 
 import (
+	"context"
 	"fmt"
 	"slices"
 
@@ -9,7 +10,7 @@ import (
 	"github.com/databricks/cli/libs/dyn"
 )
 
-func (t *translateContext) applyJobTranslations(v dyn.Value) (dyn.Value, error) {
+func (t *translateContext) applyJobTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) {
 	var err error
 
 	fallback, err := gatherFallbackPaths(v, "jobs")
@@ -38,28 +39,48 @@ func (t *translateContext) applyJobTranslations(v dyn.Value) (dyn.Value, error)
 			return dyn.InvalidValue, fmt.Errorf("unable to determine directory for job %s: %w", key, err)
 		}
 
-		rewritePatternFn, err := t.getRewritePatternFn(kind)
+		mode, err := getJobTranslateMode(kind)
 		if err != nil {
 			return dyn.InvalidValue, err
 		}
 
-		return t.rewriteRelativeTo(p, v, rewritePatternFn, dir, fallback[key])
+		opts := translateOptions{
+			Mode: mode,
+		}
+
+		// Try to rewrite the path relative to the directory of the configuration file where the value was defined.
+		nv, err := t.rewriteValue(ctx, p, v, dir, opts)
+		if err == nil {
+			return nv, nil
+		}
+
+		// If we failed to rewrite the path, try to rewrite it relative to the fallback directory.
+		// We only do this for jobs and pipelines because of the comment in [gatherFallbackPaths].
+		if fallback[key] != "" {
+			nv, nerr := t.rewriteValue(ctx, p, v, fallback[key], opts)
+			if nerr == nil {
+				// TODO: Emit a warning that this path should be rewritten.
+				return nv, nil
+			}
+		}
+
+		return dyn.InvalidValue, err
 	})
 }
 
-func (t *translateContext) getRewritePatternFn(kind paths.PathKind) (rewriteFunc, error) {
+func getJobTranslateMode(kind paths.PathKind) (TranslateMode, error) {
 	switch kind {
 	case paths.PathKindLibrary:
-		return t.translateNoOp, nil
+		return TranslateModeLocalRelative, nil
 	case paths.PathKindNotebook:
-		return t.translateNotebookPath, nil
+		return TranslateModeNotebook, nil
 	case paths.PathKindWorkspaceFile:
-		return t.translateFilePath, nil
+		return TranslateModeFile, nil
 	case paths.PathKindDirectory:
-		return t.translateDirectoryPath, nil
+		return TranslateModeDirectory, nil
 	case paths.PathKindWithPrefix:
-		return t.translateNoOpWithPrefix, nil
+		return TranslateModeLocalRelativeWithPrefix, nil
 	}
 
-	return nil, fmt.Errorf("unsupported path kind: %d", kind)
+	return TranslateMode(0), fmt.Errorf("unsupported path kind: %d", kind)
 }
diff --git a/bundle/config/mutator/translate_paths_pipelines.go b/bundle/config/mutator/translate_paths_pipelines.go
index 71a65e846..204808ff5 100644
--- a/bundle/config/mutator/translate_paths_pipelines.go
+++ b/bundle/config/mutator/translate_paths_pipelines.go
@@ -1,6 +1,7 @@
 package mutator
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/databricks/cli/libs/dyn"
@@ -8,7 +9,7 @@ import (
 
 type pipelineRewritePattern struct {
 	pattern dyn.Pattern
-	fn      rewriteFunc
+	opts    translateOptions
 }
 
 func (t *translateContext) pipelineRewritePatterns() []pipelineRewritePattern {
@@ -25,16 +26,16 @@ func (t *translateContext) pipelineRewritePatterns() []pipelineRewritePattern {
 	return []pipelineRewritePattern{
 		{
 			base.Append(dyn.Key("notebook"), dyn.Key("path")),
-			t.translateNotebookPath,
+			translateOptions{Mode: TranslateModeNotebook},
 		},
 		{
 			base.Append(dyn.Key("file"), dyn.Key("path")),
-			t.translateFilePath,
+			translateOptions{Mode: TranslateModeFile},
 		},
 	}
 }
 
-func (t *translateContext) applyPipelineTranslations(v dyn.Value) (dyn.Value, error) {
+func (t *translateContext) applyPipelineTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) {
 	var err error
 
 	fallback, err := gatherFallbackPaths(v, "pipelines")
@@ -50,7 +51,23 @@ func (t *translateContext) applyPipelineTranslations(v dyn.Value) (dyn.Value, er
 				return dyn.InvalidValue, fmt.Errorf("unable to determine directory for pipeline %s: %w", key, err)
 			}
 
-			return t.rewriteRelativeTo(p, v, rewritePattern.fn, dir, fallback[key])
+			// Try to rewrite the path relative to the directory of the configuration file where the value was defined.
+			nv, err := t.rewriteValue(ctx, p, v, dir, rewritePattern.opts)
+			if err == nil {
+				return nv, nil
+			}
+
+			// If we failed to rewrite the path, try to rewrite it relative to the fallback directory.
+			// We only do this for jobs and pipelines because of the comment in [gatherFallbackPaths].
+			if fallback[key] != "" {
+				nv, nerr := t.rewriteValue(ctx, p, v, fallback[key], rewritePattern.opts)
+				if nerr == nil {
+					// TODO: Emit a warning that this path should be rewritten.
+					return nv, nil
+				}
+			}
+
+			return dyn.InvalidValue, err
 		})
 		if err != nil {
 			return dyn.InvalidValue, err
diff --git a/bundle/tests/relative_path_with_includes/artifact_a/.gitkeep b/bundle/tests/relative_path_with_includes/artifact_a/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/bundle/tests/relative_path_with_includes/subfolder/artifact_b/.gitkeep b/bundle/tests/relative_path_with_includes/subfolder/artifact_b/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/bundle/tests/relative_path_with_includes_test.go b/bundle/tests/relative_path_with_includes_test.go
index 6e13628be..8efac0039 100644
--- a/bundle/tests/relative_path_with_includes_test.go
+++ b/bundle/tests/relative_path_with_includes_test.go
@@ -17,8 +17,8 @@ func TestRelativePathsWithIncludes(t *testing.T) {
 	diags := bundle.Apply(context.Background(), b, m)
 	assert.NoError(t, diags.Error())
 
-	assert.Equal(t, "artifact_a", b.Config.Artifacts["test_a"].Path)
-	assert.Equal(t, filepath.Join("subfolder", "artifact_b"), b.Config.Artifacts["test_b"].Path)
+	assert.Equal(t, filepath.Join(b.SyncRootPath, "artifact_a"), b.Config.Artifacts["test_a"].Path)
+	assert.Equal(t, filepath.Join(b.SyncRootPath, "subfolder", "artifact_b"), b.Config.Artifacts["test_b"].Path)
 
 	assert.ElementsMatch(
 		t,

From 2b452973f393b58d1565820d3d46aaa9b67f4305 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 14 Jan 2025 11:56:38 +0100
Subject: [PATCH 066/247] Enable linter 'unconvert' and fix the issues found
 (#2136)

---
 .golangci.yaml                            | 1 +
 bundle/apps/interpolate_variables_test.go | 2 +-
 libs/cmdio/logger.go                      | 2 +-
 libs/dyn/value_underlying.go              | 2 +-
 libs/notebook/detect.go                   | 2 +-
 5 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.golangci.yaml b/.golangci.yaml
index 07a6afdc5..9711a70af 100644
--- a/.golangci.yaml
+++ b/.golangci.yaml
@@ -15,6 +15,7 @@ linters:
     - intrange
     - mirror
     - perfsprint
+    - unconvert
 linters-settings:
   govet:
     enable-all: true
diff --git a/bundle/apps/interpolate_variables_test.go b/bundle/apps/interpolate_variables_test.go
index a2909006f..b6c424a95 100644
--- a/bundle/apps/interpolate_variables_test.go
+++ b/bundle/apps/interpolate_variables_test.go
@@ -44,6 +44,6 @@ func TestAppInterpolateVariables(t *testing.T) {
 
 	diags := bundle.Apply(context.Background(), b, InterpolateVariables())
 	require.Empty(t, diags)
-	require.Equal(t, []any([]any{map[string]any{"name": "JOB_ID", "value": "123"}}), b.Config.Resources.Apps["my_app_1"].Config["env"])
+	require.Equal(t, []any{map[string]any{"name": "JOB_ID", "value": "123"}}, b.Config.Resources.Apps["my_app_1"].Config["env"])
 	require.Nil(t, b.Config.Resources.Apps["my_app_2"].Config)
 }
diff --git a/libs/cmdio/logger.go b/libs/cmdio/logger.go
index 7edad5bf0..48b76ce42 100644
--- a/libs/cmdio/logger.go
+++ b/libs/cmdio/logger.go
@@ -189,7 +189,7 @@ func (l *Logger) writeJson(event Event) {
 		// we panic because there we cannot catch this in jobs.RunNowAndWait
 		panic(err)
 	}
-	_, _ = l.Writer.Write([]byte(b))
+	_, _ = l.Writer.Write(b)
 	_, _ = l.Writer.Write([]byte("\n"))
 }
 
diff --git a/libs/dyn/value_underlying.go b/libs/dyn/value_underlying.go
index 0a867375d..a33ecd38e 100644
--- a/libs/dyn/value_underlying.go
+++ b/libs/dyn/value_underlying.go
@@ -81,7 +81,7 @@ func (v Value) AsInt() (int64, bool) {
 	case int32:
 		return int64(vv), true
 	case int64:
-		return int64(vv), true
+		return vv, true
 	default:
 		return 0, false
 	}
diff --git a/libs/notebook/detect.go b/libs/notebook/detect.go
index 40c850945..579cc1de3 100644
--- a/libs/notebook/detect.go
+++ b/libs/notebook/detect.go
@@ -47,7 +47,7 @@ func (f file) close() error {
 func (f file) readHeader() (string, error) {
 	// Scan header line with some padding.
 	buf := make([]byte, headerLength)
-	n, err := f.f.Read([]byte(buf))
+	n, err := f.f.Read(buf)
 	if err != nil && err != io.EOF {
 		return "", err
 	}

From 98a1e73a0f97a5fb7719a44edee0a5e81947e9a6 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 14 Jan 2025 12:00:38 +0100
Subject: [PATCH 067/247] Simplify replacements logic for golden files (#2132)

## Changes
- Do not sort, use fixed order of replacements.

## Tests
Existing tests.
---
 libs/testdiff/golden.go      | 39 +++++++++++++++---------------------
 libs/testdiff/golden_test.go | 13 ------------
 2 files changed, 16 insertions(+), 36 deletions(-)
 delete mode 100644 libs/testdiff/golden_test.go

diff --git a/libs/testdiff/golden.go b/libs/testdiff/golden.go
index 08d1e9608..0ed38e686 100644
--- a/libs/testdiff/golden.go
+++ b/libs/testdiff/golden.go
@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"os"
 	"regexp"
-	"slices"
 	"strings"
 	"testing"
 
@@ -17,6 +16,10 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+const (
+	testerName = "$USERNAME"
+)
+
 var OverwriteMode = false
 
 func init() {
@@ -165,7 +168,7 @@ func PrepareReplacements(t testutil.TestingT, r *ReplacementsContext, w *databri
 	r.Set(w.Config.AzureResourceID, "$DATABRICKS_AZURE_RESOURCE_ID")
 	r.Set(w.Config.AzureClientSecret, "$ARM_CLIENT_SECRET")
 	// r.Set(w.Config.AzureClientID, "$ARM_CLIENT_ID")
-	r.Set(w.Config.AzureClientID, "$USERNAME")
+	r.Set(w.Config.AzureClientID, testerName)
 	r.Set(w.Config.AzureTenantID, "$ARM_TENANT_ID")
 	r.Set(w.Config.ActionsIDTokenRequestURL, "$ACTIONS_ID_TOKEN_REQUEST_URL")
 	r.Set(w.Config.ActionsIDTokenRequestToken, "$ACTIONS_ID_TOKEN_REQUEST_TOKEN")
@@ -181,24 +184,20 @@ func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam.
 	t.Helper()
 	// There could be exact matches or overlap between different name fields, so sort them by length
 	// to ensure we match the largest one first and map them all to the same token
-	names := []string{
-		u.DisplayName,
-		u.UserName,
-		iamutil.GetShortUserName(&u),
-	}
-	if u.Name != nil {
-		names = append(names, u.Name.FamilyName)
-		names = append(names, u.Name.GivenName)
-	}
-	for _, val := range u.Emails {
-		names = append(names, val.Value)
-	}
-	stableSortReverseLength(names)
 
-	for _, name := range names {
-		r.Set(name, "$USERNAME")
+	r.Set(u.UserName, testerName)
+	r.Set(u.DisplayName, testerName)
+	if u.Name != nil {
+		r.Set(u.Name.FamilyName, testerName)
+		r.Set(u.Name.GivenName, testerName)
 	}
 
+	for _, val := range u.Emails {
+		r.Set(val.Value, testerName)
+	}
+
+	r.Set(iamutil.GetShortUserName(&u), testerName)
+
 	for ind, val := range u.Groups {
 		r.Set(val.Value, fmt.Sprintf("$USER.Groups[%d]", ind))
 	}
@@ -210,12 +209,6 @@ func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam.
 	}
 }
 
-func stableSortReverseLength(strs []string) {
-	slices.SortStableFunc(strs, func(a, b string) int {
-		return len(b) - len(a)
-	})
-}
-
 func NormalizeNewlines(input string) string {
 	output := strings.ReplaceAll(input, "\r\n", "\n")
 	return strings.ReplaceAll(output, "\r", "\n")
diff --git a/libs/testdiff/golden_test.go b/libs/testdiff/golden_test.go
deleted file mode 100644
index 0fc32be21..000000000
--- a/libs/testdiff/golden_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package testdiff
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestSort(t *testing.T) {
-	input := []string{"a", "bc", "cd"}
-	stableSortReverseLength(input)
-	assert.Equal(t, []string{"bc", "cd", "a"}, input)
-}

From fe31e4d02ea0817e3b80ad3a8c0bdf85d3847675 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Tue, 14 Jan 2025 14:24:22 +0100
Subject: [PATCH 068/247] Fixed a typo in TestDeployBundleWithApp test (#2138)

## Changes
Fixed a typo in TestDeployBundleWithApp test

## Tests
```
   helpers_test.go:148: stderr: Destroy complete!
--- PASS: TestDeployBundleWithApp (647.51s)
PASS
coverage: [no statements]
ok      github.com/databricks/cli/integration/bundle    647.985s        coverage: [no statements]
```
---
 integration/bundle/apps_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/bundle/apps_test.go b/integration/bundle/apps_test.go
index f15d8aabc..2da764ed8 100644
--- a/integration/bundle/apps_test.go
+++ b/integration/bundle/apps_test.go
@@ -27,7 +27,7 @@ func TestDeployBundleWithApp(t *testing.T) {
 	}
 
 	uniqueId := uuid.New().String()
-	appId := "app-%s" + uuid.New().String()[0:8]
+	appId := "app-" + uuid.New().String()[0:8]
 	nodeTypeId := testutil.GetCloud(t).NodeTypeID()
 	instancePoolId := env.Get(ctx, "TEST_INSTANCE_POOL_ID")
 

From 2ae2b7e8c8e13d844280b82c9452b040d123ae7f Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 14 Jan 2025 14:50:28 +0100
Subject: [PATCH 069/247] Enable acceptance tests for manually running against
 the cloud (#2120)

## Changes
- If CLOUD_ENV variable is set, acceptance will no longer set up server
& override DATABRICKS_HOST/DATABRICKS_TOKEN/HOME env vars.
- I've updated replacements logic in testdiff to use tester /
tester@databricks.com convention.

## Tests
Manually running current acceptance tests against dogfood on my laptop I
get all test pass except for 2 failures.

```
    --- FAIL: TestAccept/bundle/variables/env_overrides (0.09s)
    --- FAIL: TestAccept/bundle/variables/resolve-builtin (1.30s)
```
---
 acceptance/acceptance_test.go                 | 36 +++++++++++++------
 .../bundle/override/job_cluster/output.txt    |  4 +--
 .../override/job_cluster_var/output.txt       | 12 +++----
 .../bundle/override/job_tasks/output.txt      |  4 +--
 .../override/merge-string-map/output.txt      |  4 +--
 .../override/pipeline_cluster/output.txt      |  4 +--
 .../bundle/variables/complex/out.default.json |  2 +-
 .../bundle/variables/complex/out.dev.json     |  2 +-
 .../complex_multiple_files/output.txt         |  2 +-
 acceptance/bundle/variables/empty/output.txt  |  4 +--
 .../bundle/variables/env_overrides/output.txt |  4 +--
 .../variables/resolve-builtin/output.txt      |  4 +--
 .../bundle/variables/vanilla/output.txt       |  4 +--
 .../variable_overrides_in_target/output.txt   |  8 ++---
 14 files changed, 55 insertions(+), 39 deletions(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index b9fb219dc..63dd95f1f 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -1,6 +1,7 @@
 package acceptance_test
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -17,6 +18,7 @@ import (
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/env"
 	"github.com/databricks/cli/libs/testdiff"
+	"github.com/databricks/databricks-sdk-go"
 	"github.com/stretchr/testify/require"
 )
 
@@ -45,19 +47,33 @@ func TestAccept(t *testing.T) {
 	// Make helper scripts available
 	t.Setenv("PATH", fmt.Sprintf("%s%c%s", filepath.Join(cwd, "bin"), os.PathListSeparator, os.Getenv("PATH")))
 
-	server := StartServer(t)
-	AddHandlers(server)
-	// Redirect API access to local server:
-	t.Setenv("DATABRICKS_HOST", fmt.Sprintf("http://127.0.0.1:%d", server.Port))
-	t.Setenv("DATABRICKS_TOKEN", "dapi1234")
-
-	homeDir := t.TempDir()
-	// Do not read user's ~/.databrickscfg
-	t.Setenv(env.HomeEnvVar(), homeDir)
-
 	repls := testdiff.ReplacementsContext{}
 	repls.Set(execPath, "$CLI")
 
+	ctx := context.Background()
+	cloudEnv := os.Getenv("CLOUD_ENV")
+
+	if cloudEnv == "" {
+		server := StartServer(t)
+		AddHandlers(server)
+		// Redirect API access to local server:
+		t.Setenv("DATABRICKS_HOST", fmt.Sprintf("http://127.0.0.1:%d", server.Port))
+		t.Setenv("DATABRICKS_TOKEN", "dapi1234")
+
+		homeDir := t.TempDir()
+		// Do not read user's ~/.databrickscfg
+		t.Setenv(env.HomeEnvVar(), homeDir)
+	}
+
+	workspaceClient, err := databricks.NewWorkspaceClient()
+	require.NoError(t, err)
+
+	user, err := workspaceClient.CurrentUser.Me(ctx)
+	require.NoError(t, err)
+	require.NotNil(t, user)
+	testdiff.PrepareReplacementsUser(t, &repls, *user)
+	testdiff.PrepareReplacements(t, &repls, workspaceClient)
+
 	testDirs := getTests(t)
 	require.NotEmpty(t, testDirs)
 	for _, dir := range testDirs {
diff --git a/acceptance/bundle/override/job_cluster/output.txt b/acceptance/bundle/override/job_cluster/output.txt
index 947d19032..ff6e8316e 100644
--- a/acceptance/bundle/override/job_cluster/output.txt
+++ b/acceptance/bundle/override/job_cluster/output.txt
@@ -4,7 +4,7 @@
   "foo": {
     "deployment": {
       "kind": "BUNDLE",
-      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development/state/metadata.json"
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/development/state/metadata.json"
     },
     "edit_mode": "UI_LOCKED",
     "format": "MULTI_TASK",
@@ -32,7 +32,7 @@
   "foo": {
     "deployment": {
       "kind": "BUNDLE",
-      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging/state/metadata.json"
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/staging/state/metadata.json"
     },
     "edit_mode": "UI_LOCKED",
     "format": "MULTI_TASK",
diff --git a/acceptance/bundle/override/job_cluster_var/output.txt b/acceptance/bundle/override/job_cluster_var/output.txt
index cb76de5a8..0b19e5eb2 100644
--- a/acceptance/bundle/override/job_cluster_var/output.txt
+++ b/acceptance/bundle/override/job_cluster_var/output.txt
@@ -4,7 +4,7 @@
   "foo": {
     "deployment": {
       "kind": "BUNDLE",
-      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development/state/metadata.json"
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/development/state/metadata.json"
     },
     "edit_mode": "UI_LOCKED",
     "format": "MULTI_TASK",
@@ -31,8 +31,8 @@
 Name: override_job_cluster
 Target: development
 Workspace:
-  User: tester@databricks.com
-  Path: /Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/override_job_cluster/development
 
 Validation OK!
 
@@ -41,7 +41,7 @@ Validation OK!
   "foo": {
     "deployment": {
       "kind": "BUNDLE",
-      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging/state/metadata.json"
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/staging/state/metadata.json"
     },
     "edit_mode": "UI_LOCKED",
     "format": "MULTI_TASK",
@@ -68,7 +68,7 @@ Validation OK!
 Name: override_job_cluster
 Target: staging
 Workspace:
-  User: tester@databricks.com
-  Path: /Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/override_job_cluster/staging
 
 Validation OK!
diff --git a/acceptance/bundle/override/job_tasks/output.txt b/acceptance/bundle/override/job_tasks/output.txt
index 0bb0b1812..915351d4e 100644
--- a/acceptance/bundle/override/job_tasks/output.txt
+++ b/acceptance/bundle/override/job_tasks/output.txt
@@ -69,8 +69,8 @@ Error: file ./test1.py not found
 Name: override_job_tasks
 Target: staging
 Workspace:
-  User: tester@databricks.com
-  Path: /Workspace/Users/tester@databricks.com/.bundle/override_job_tasks/staging
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/override_job_tasks/staging
 
 Found 1 error
 
diff --git a/acceptance/bundle/override/merge-string-map/output.txt b/acceptance/bundle/override/merge-string-map/output.txt
index 986da8174..b566aa07f 100644
--- a/acceptance/bundle/override/merge-string-map/output.txt
+++ b/acceptance/bundle/override/merge-string-map/output.txt
@@ -21,7 +21,7 @@ Warning: expected map, found string
 Name: merge-string-map
 Target: dev
 Workspace:
-  User: tester@databricks.com
-  Path: /Workspace/Users/tester@databricks.com/.bundle/merge-string-map/dev
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/merge-string-map/dev
 
 Found 1 warning
diff --git a/acceptance/bundle/override/pipeline_cluster/output.txt b/acceptance/bundle/override/pipeline_cluster/output.txt
index 81bf58180..8babed0ec 100644
--- a/acceptance/bundle/override/pipeline_cluster/output.txt
+++ b/acceptance/bundle/override/pipeline_cluster/output.txt
@@ -14,7 +14,7 @@
     ],
     "deployment": {
       "kind": "BUNDLE",
-      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_pipeline_cluster/development/state/metadata.json"
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_pipeline_cluster/development/state/metadata.json"
     },
     "name": "job",
     "permissions": []
@@ -36,7 +36,7 @@
     ],
     "deployment": {
       "kind": "BUNDLE",
-      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_pipeline_cluster/staging/state/metadata.json"
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_pipeline_cluster/staging/state/metadata.json"
     },
     "name": "job",
     "permissions": []
diff --git a/acceptance/bundle/variables/complex/out.default.json b/acceptance/bundle/variables/complex/out.default.json
index 6454562a6..a1ccd52bc 100644
--- a/acceptance/bundle/variables/complex/out.default.json
+++ b/acceptance/bundle/variables/complex/out.default.json
@@ -4,7 +4,7 @@
       "my_job": {
         "deployment": {
           "kind": "BUNDLE",
-          "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/complex-variables/default/state/metadata.json"
+          "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/complex-variables/default/state/metadata.json"
         },
         "edit_mode": "UI_LOCKED",
         "format": "MULTI_TASK",
diff --git a/acceptance/bundle/variables/complex/out.dev.json b/acceptance/bundle/variables/complex/out.dev.json
index cede5feb2..bb939091b 100644
--- a/acceptance/bundle/variables/complex/out.dev.json
+++ b/acceptance/bundle/variables/complex/out.dev.json
@@ -4,7 +4,7 @@
       "my_job": {
         "deployment": {
           "kind": "BUNDLE",
-          "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/complex-variables/dev/state/metadata.json"
+          "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/complex-variables/dev/state/metadata.json"
         },
         "edit_mode": "UI_LOCKED",
         "format": "MULTI_TASK",
diff --git a/acceptance/bundle/variables/complex_multiple_files/output.txt b/acceptance/bundle/variables/complex_multiple_files/output.txt
index e87b8df11..ec2cad1ce 100644
--- a/acceptance/bundle/variables/complex_multiple_files/output.txt
+++ b/acceptance/bundle/variables/complex_multiple_files/output.txt
@@ -4,7 +4,7 @@
       "my_job": {
         "deployment": {
           "kind": "BUNDLE",
-          "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/complex-variables-multiple-files/dev/state/metadata.json"
+          "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/complex-variables-multiple-files/dev/state/metadata.json"
         },
         "edit_mode": "UI_LOCKED",
         "format": "MULTI_TASK",
diff --git a/acceptance/bundle/variables/empty/output.txt b/acceptance/bundle/variables/empty/output.txt
index c3f0af130..261635920 100644
--- a/acceptance/bundle/variables/empty/output.txt
+++ b/acceptance/bundle/variables/empty/output.txt
@@ -3,8 +3,8 @@ Error: no value assigned to required variable a. Assignment can be done through
 Name: empty${var.a}
 Target: default
 Workspace:
-  User: tester@databricks.com
-  Path: /Workspace/Users/tester@databricks.com/.bundle/empty${var.a}/default
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/empty${var.a}/default
 
 Found 1 error
 
diff --git a/acceptance/bundle/variables/env_overrides/output.txt b/acceptance/bundle/variables/env_overrides/output.txt
index e8fb99938..f42f82211 100644
--- a/acceptance/bundle/variables/env_overrides/output.txt
+++ b/acceptance/bundle/variables/env_overrides/output.txt
@@ -14,8 +14,8 @@ Error: no value assigned to required variable b. Assignment can be done through
 Name: test bundle
 Target: env-missing-a-required-variable-assignment
 Workspace:
-  User: tester@databricks.com
-  Path: /Workspace/Users/tester@databricks.com/.bundle/test bundle/env-missing-a-required-variable-assignment
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/test bundle/env-missing-a-required-variable-assignment
 
 Found 1 error
 
diff --git a/acceptance/bundle/variables/resolve-builtin/output.txt b/acceptance/bundle/variables/resolve-builtin/output.txt
index 2f58abd8a..f060c472e 100644
--- a/acceptance/bundle/variables/resolve-builtin/output.txt
+++ b/acceptance/bundle/variables/resolve-builtin/output.txt
@@ -1,8 +1,8 @@
 {
   "artifact_path": "TestResolveVariableReferences/bar/artifacts",
   "current_user": {
-    "short_name": "tester",
-    "userName": "tester@databricks.com"
+    "short_name": "$USERNAME",
+    "userName": "$USERNAME"
   },
   "file_path": "TestResolveVariableReferences/bar/baz",
   "resource_path": "TestResolveVariableReferences/bar/resources",
diff --git a/acceptance/bundle/variables/vanilla/output.txt b/acceptance/bundle/variables/vanilla/output.txt
index 69b358a3f..1d88bd060 100644
--- a/acceptance/bundle/variables/vanilla/output.txt
+++ b/acceptance/bundle/variables/vanilla/output.txt
@@ -8,8 +8,8 @@ Error: no value assigned to required variable b. Assignment can be done through
 Name: ${var.a} ${var.b}
 Target: default
 Workspace:
-  User: tester@databricks.com
-  Path: /Workspace/Users/tester@databricks.com/.bundle/${var.a} ${var.b}/default
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/${var.a} ${var.b}/default
 
 Found 1 error
 
diff --git a/acceptance/bundle/variables/variable_overrides_in_target/output.txt b/acceptance/bundle/variables/variable_overrides_in_target/output.txt
index de193f5b6..8998b691d 100644
--- a/acceptance/bundle/variables/variable_overrides_in_target/output.txt
+++ b/acceptance/bundle/variables/variable_overrides_in_target/output.txt
@@ -12,7 +12,7 @@
       "continuous": true,
       "deployment": {
         "kind": "BUNDLE",
-        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/foobar/use-default-variable-values/state/metadata.json"
+        "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/foobar/use-default-variable-values/state/metadata.json"
       },
       "name": "a_string",
       "permissions": []
@@ -33,7 +33,7 @@
       "continuous": true,
       "deployment": {
         "kind": "BUNDLE",
-        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/foobar/override-string-variable/state/metadata.json"
+        "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/foobar/override-string-variable/state/metadata.json"
       },
       "name": "overridden_string",
       "permissions": []
@@ -54,7 +54,7 @@
       "continuous": true,
       "deployment": {
         "kind": "BUNDLE",
-        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/foobar/override-int-variable/state/metadata.json"
+        "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/foobar/override-int-variable/state/metadata.json"
       },
       "name": "a_string",
       "permissions": []
@@ -75,7 +75,7 @@
       "continuous": false,
       "deployment": {
         "kind": "BUNDLE",
-        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/foobar/override-both-bool-and-string-variables/state/metadata.json"
+        "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/foobar/override-both-bool-and-string-variables/state/metadata.json"
       },
       "name": "overridden_string",
       "permissions": []

From a5e09ab28a3728252672bcaac3ffd4cdffac36f7 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 14 Jan 2025 15:19:00 +0100
Subject: [PATCH 070/247] Coverage for acceptance tests (#2123)

## Changes

Add two new make commands:
- make acc-cover: runs acceptance tests and outputs
coverage-acceptance.txt
- make acc-showcover: show coverage-acceptance.txt locally in browser

Using the GOCOVERDIR functionality:
https://go.dev/blog/integration-test-coverage

This works, but there are a couple of issues encountered:
- GOCOVERDIR does not play well with regular "go test -cover". Once this
fixed, we can simplify the code and have 'make cover' output coverage
for everything at once. We can also probably get rid of CLI_GOCOVERDIR.
https://github.com/golang/go/issues/66225
- When running tests in parallel to the same directory there is rare
conflict on writing covmeta file. For this reason each tests writes
coverage to their own directory which is then merged together by 'make
acc-cover'.

<!-- Summary of your changes that are easy to understand --


## Tests
Manually running the new make commands.
---
 .gitignore                    |  1 +
 Makefile                      | 13 ++++++++++++-
 acceptance/acceptance_test.go | 30 ++++++++++++++++++++++++++----
 3 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/.gitignore b/.gitignore
index edd1409ae..274c1341a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,6 +20,7 @@ dist/
 
 *.log
 coverage.txt
+coverage-acceptance.txt
 
 __pycache__
 *.pyc
diff --git a/Makefile b/Makefile
index 2c84d88ba..4e5067b5d 100644
--- a/Makefile
+++ b/Makefile
@@ -25,6 +25,17 @@ cover:
 showcover:
 	go tool cover -html=coverage.txt
 
+acc-cover:
+	rm -fr ./acceptance/build/cover/
+	CLI_GOCOVERDIR=build/cover go test ./acceptance
+	rm -fr ./acceptance/build/cover-merged/
+	mkdir -p acceptance/build/cover-merged/
+	go tool covdata merge -i $$(printf '%s,' acceptance/build/cover/* | sed 's/,$$//') -o acceptance/build/cover-merged/
+	go tool covdata textfmt -i acceptance/build/cover-merged -o coverage-acceptance.txt
+
+acc-showcover:
+	go tool cover -html=coverage-acceptance.txt
+
 build: vendor
 	go build -mod vendor
 
@@ -45,4 +56,4 @@ integration:
 integration-short:
 	$(INTEGRATION) -short
 
-.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short
+.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover
diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 63dd95f1f..4b5146503 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -40,7 +40,16 @@ func TestAccept(t *testing.T) {
 	cwd, err := os.Getwd()
 	require.NoError(t, err)
 
-	execPath := BuildCLI(t, cwd)
+	coverDir := os.Getenv("CLI_GOCOVERDIR")
+
+	if coverDir != "" {
+		require.NoError(t, os.MkdirAll(coverDir, os.ModePerm))
+		coverDir, err = filepath.Abs(coverDir)
+		require.NoError(t, err)
+		t.Logf("Writing coverage to %s", coverDir)
+	}
+
+	execPath := BuildCLI(t, cwd, coverDir)
 	// $CLI is what test scripts are using
 	t.Setenv("CLI", execPath)
 
@@ -76,10 +85,11 @@ func TestAccept(t *testing.T) {
 
 	testDirs := getTests(t)
 	require.NotEmpty(t, testDirs)
+
 	for _, dir := range testDirs {
 		t.Run(dir, func(t *testing.T) {
 			t.Parallel()
-			runTest(t, dir, repls)
+			runTest(t, dir, coverDir, repls)
 		})
 	}
 }
@@ -104,7 +114,7 @@ func getTests(t *testing.T) []string {
 	return testDirs
 }
 
-func runTest(t *testing.T, dir string, repls testdiff.ReplacementsContext) {
+func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsContext) {
 	var tmpDir string
 	var err error
 	if KeepTmp {
@@ -127,6 +137,15 @@ func runTest(t *testing.T, dir string, repls testdiff.ReplacementsContext) {
 
 	args := []string{"bash", "-euo", "pipefail", EntryPointScript}
 	cmd := exec.Command(args[0], args[1:]...)
+	if coverDir != "" {
+		// Creating individual coverage directory for each test, because writing to the same one
+		// results in sporadic failures like this one (only if tests are running in parallel):
+		// +error: coverage meta-data emit failed: writing ... rename .../tmp.covmeta.b3f... .../covmeta.b3f2c...: no such file or directory
+		coverDir = filepath.Join(coverDir, strings.ReplaceAll(dir, string(os.PathSeparator), "--"))
+		err := os.MkdirAll(coverDir, os.ModePerm)
+		require.NoError(t, err)
+		cmd.Env = append(os.Environ(), "GOCOVERDIR="+coverDir)
+	}
 	cmd.Dir = tmpDir
 	outB, err := cmd.CombinedOutput()
 
@@ -226,7 +245,7 @@ func readMergedScriptContents(t *testing.T, dir string) string {
 	return strings.Join(prepares, "\n")
 }
 
-func BuildCLI(t *testing.T, cwd string) string {
+func BuildCLI(t *testing.T, cwd, coverDir string) string {
 	execPath := filepath.Join(cwd, "build", "databricks")
 	if runtime.GOOS == "windows" {
 		execPath += ".exe"
@@ -234,6 +253,9 @@ func BuildCLI(t *testing.T, cwd string) string {
 
 	start := time.Now()
 	args := []string{"go", "build", "-mod", "vendor", "-o", execPath}
+	if coverDir != "" {
+		args = append(args, "-cover")
+	}
 	cmd := exec.Command(args[0], args[1:]...)
 	cmd.Dir = ".."
 	out, err := cmd.CombinedOutput()

From ccb2599b422c7c208d13ae32be0748bc48f67e49 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 14 Jan 2025 16:38:20 +0100
Subject: [PATCH 071/247] Add complex-transitive-deeper acceptance test (#2140)

Extension of complex-transitive test that shows an error instead of
simply failing to interpolate.
---
 .../complex-transitive-deeper/databricks.yml  | 22 +++++++++++++++++++
 .../complex-transitive-deeper/output.txt      |  7 ++++++
 .../complex-transitive-deeper/script          |  2 ++
 3 files changed, 31 insertions(+)
 create mode 100644 acceptance/bundle/variables/complex-transitive-deeper/databricks.yml
 create mode 100644 acceptance/bundle/variables/complex-transitive-deeper/output.txt
 create mode 100644 acceptance/bundle/variables/complex-transitive-deeper/script

diff --git a/acceptance/bundle/variables/complex-transitive-deeper/databricks.yml b/acceptance/bundle/variables/complex-transitive-deeper/databricks.yml
new file mode 100644
index 000000000..3f9bea464
--- /dev/null
+++ b/acceptance/bundle/variables/complex-transitive-deeper/databricks.yml
@@ -0,0 +1,22 @@
+bundle:
+  name: complex-transitive-deeper
+
+variables:
+  catalog_1:
+    default:
+      name: hive_metastore
+  catalog:
+    default: ${var.catalog_1}
+  spark_conf:
+    default:
+      "spark.databricks.sql.initial.catalog.name": ${var.catalog.name}
+  etl_cluster_config:
+    type: complex
+    default:
+      spark_version: 14.3.x-scala2.12
+      runtime_engine: PHOTON
+      spark_conf: ${var.spark_conf}
+
+resources:
+  clusters:
+    my_cluster: ${var.etl_cluster_config}
diff --git a/acceptance/bundle/variables/complex-transitive-deeper/output.txt b/acceptance/bundle/variables/complex-transitive-deeper/output.txt
new file mode 100644
index 000000000..3bedbfb9a
--- /dev/null
+++ b/acceptance/bundle/variables/complex-transitive-deeper/output.txt
@@ -0,0 +1,7 @@
+Error: expected a map to index "variables.catalog.value.name", found string
+
+{
+  "my_cluster": "${var.etl_cluster_config}"
+}
+
+Exit code: 1
diff --git a/acceptance/bundle/variables/complex-transitive-deeper/script b/acceptance/bundle/variables/complex-transitive-deeper/script
new file mode 100644
index 000000000..d4fb404b1
--- /dev/null
+++ b/acceptance/bundle/variables/complex-transitive-deeper/script
@@ -0,0 +1,2 @@
+# Currently, this errors instead of interpolating variables
+$CLI bundle validate -o json | jq '.resources.clusters'

From fca6abdfac99b048810f8e97aeadbd178cb84e88 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 14 Jan 2025 16:50:55 +0100
Subject: [PATCH 072/247] Golden files: always include JSON-ed string (#2142)

## Changes
Always include both verbatim and json version of replacement.

This helps when the string in question contains \\ or other chars that
need to be quoted.

Needed for https://github.com/databricks/cli/pull/2118

## Tests
Existing tests.
---
 libs/testdiff/golden.go | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/libs/testdiff/golden.go b/libs/testdiff/golden.go
index 0ed38e686..78973b468 100644
--- a/libs/testdiff/golden.go
+++ b/libs/testdiff/golden.go
@@ -2,6 +2,7 @@ package testdiff
 
 import (
 	"context"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"os"
@@ -122,6 +123,19 @@ func (r *ReplacementsContext) Set(old, new string) {
 	if old == "" || new == "" {
 		return
 	}
+
+	// Always include both verbatim and json version of replacement.
+	// This helps when the string in question contains \ or other chars that need to be quoted.
+	// In that case we cannot rely that json(old) == '"{old}"' and need to add it explicitly.
+
+	encodedNew, err := json.Marshal(new)
+	if err != nil {
+		encodedOld, err := json.Marshal(old)
+		if err != nil {
+			r.Repls = append(r.Repls, Replacement{Old: string(encodedOld), New: string(encodedNew)})
+		}
+	}
+
 	r.Repls = append(r.Repls, Replacement{Old: old, New: new})
 }
 

From 72e677d0ac4b01b3571853bd90170dbb25e828bd Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 14 Jan 2025 16:02:34 +0000
Subject: [PATCH 073/247] Bump github.com/databricks/databricks-sdk-go from
 0.54.0 to 0.55.0 (#2126)

Bumps
[github.com/databricks/databricks-sdk-go](https://github.com/databricks/databricks-sdk-go)
from 0.54.0 to 0.55.0.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/databricks/databricks-sdk-go/releases">github.com/databricks/databricks-sdk-go's
releases</a>.</em></p>
<blockquote>
<h2>v0.55.0</h2>
<h3>Internal Changes</h3>
<ul>
<li>Bump staticcheck to 0.5.1 and add go 1.23 test coverage (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1106">#1106</a>).</li>
<li>Bump x/net, x/crypto dependencies (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1107">#1107</a>).</li>
<li>Create custom codeql.yml (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1114">#1114</a>).</li>
<li>Decouple serving and oauth2 package (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1110">#1110</a>).</li>
<li>Migrate workflows that need write access to use hosted runners (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1112">#1112</a>).</li>
<li>Move package credentials in config (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1115">#1115</a>).</li>
<li>Update Queries test (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1104">#1104</a>).</li>
</ul>
<h3>API Changes:</h3>
<ul>
<li>Added <code>NoCompute</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/apps#CreateAppRequest">apps.CreateAppRequest</a>.</li>
<li>Added <code>HasMore</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#BaseJob">jobs.BaseJob</a>.</li>
<li>Added <code>HasMore</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#BaseRun">jobs.BaseRun</a>.</li>
<li>Added <code>PageToken</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#GetJobRequest">jobs.GetJobRequest</a>.</li>
<li>Added <code>HasMore</code> and <code>NextPageToken</code> fields for
<a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#Job">jobs.Job</a>.</li>
<li>Added <code>HasMore</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#Run">jobs.Run</a>.</li>
<li>Added <code>RunAs</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#CreatePipeline">pipelines.CreatePipeline</a>.</li>
<li>Added <code>RunAs</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#EditPipeline">pipelines.EditPipeline</a>.</li>
<li>Added <code>AuthorizationDetails</code> and <code>EndpointUrl</code>
fields for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#DataPlaneInfo">serving.DataPlaneInfo</a>.</li>
<li>[Breaking] Changed <code>Update</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/oauth2#AccountFederationPolicyAPI">a.AccountFederationPolicy</a>
account-level service with new required argument order.</li>
<li>[Breaking] Changed <code>Update</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/oauth2#ServicePrincipalFederationPolicyAPI">a.ServicePrincipalFederationPolicy</a>
account-level service with new required argument order.</li>
<li>Changed <code>UpdateMask</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/oauth2#UpdateAccountFederationPolicyRequest">oauth2.UpdateAccountFederationPolicyRequest</a>
to no longer be required.</li>
<li>Changed <code>UpdateMask</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/oauth2#UpdateServicePrincipalFederationPolicyRequest">oauth2.UpdateServicePrincipalFederationPolicyRequest</a>
to no longer be required.</li>
<li>[Breaking] Changed <code>DaysOfWeek</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#RestartWindow">pipelines.RestartWindow</a>
to type <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#DayOfWeekList">pipelines.DayOfWeekList</a>.</li>
</ul>
<p>OpenAPI SHA: 779817ed8d63031f5ea761fbd25ee84f38feec0d, Date:
2025-01-08</p>
</blockquote>
</details>
<details>
<summary>Changelog</summary>
<p><em>Sourced from <a
href="https://github.com/databricks/databricks-sdk-go/blob/main/CHANGELOG.md">github.com/databricks/databricks-sdk-go's
changelog</a>.</em></p>
<blockquote>
<h2>[Release] Release v0.55.0</h2>
<h3>Internal Changes</h3>
<ul>
<li>Bump staticcheck to 0.5.1 and add go 1.23 test coverage (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1106">#1106</a>).</li>
<li>Bump x/net, x/crypto dependencies (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1107">#1107</a>).</li>
<li>Create custom codeql.yml (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1114">#1114</a>).</li>
<li>Decouple serving and oauth2 package (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1110">#1110</a>).</li>
<li>Migrate workflows that need write access to use hosted runners (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1112">#1112</a>).</li>
<li>Move package credentials in config (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1115">#1115</a>).</li>
<li>Update Queries test (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1104">#1104</a>).</li>
</ul>
<h3>API Changes:</h3>
<ul>
<li>Added <code>NoCompute</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/apps#CreateAppRequest">apps.CreateAppRequest</a>.</li>
<li>Added <code>HasMore</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#BaseJob">jobs.BaseJob</a>.</li>
<li>Added <code>HasMore</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#BaseRun">jobs.BaseRun</a>.</li>
<li>Added <code>PageToken</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#GetJobRequest">jobs.GetJobRequest</a>.</li>
<li>Added <code>HasMore</code> and <code>NextPageToken</code> fields for
<a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#Job">jobs.Job</a>.</li>
<li>Added <code>HasMore</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#Run">jobs.Run</a>.</li>
<li>Added <code>RunAs</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#CreatePipeline">pipelines.CreatePipeline</a>.</li>
<li>Added <code>RunAs</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#EditPipeline">pipelines.EditPipeline</a>.</li>
<li>Added <code>AuthorizationDetails</code> and <code>EndpointUrl</code>
fields for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#DataPlaneInfo">serving.DataPlaneInfo</a>.</li>
<li>[Breaking] Changed <code>Update</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/oauth2#AccountFederationPolicyAPI">a.AccountFederationPolicy</a>
account-level service with new required argument order.</li>
<li>[Breaking] Changed <code>Update</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/oauth2#ServicePrincipalFederationPolicyAPI">a.ServicePrincipalFederationPolicy</a>
account-level service with new required argument order.</li>
<li>Changed <code>UpdateMask</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/oauth2#UpdateAccountFederationPolicyRequest">oauth2.UpdateAccountFederationPolicyRequest</a>
to no longer be required.</li>
<li>Changed <code>UpdateMask</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/oauth2#UpdateServicePrincipalFederationPolicyRequest">oauth2.UpdateServicePrincipalFederationPolicyRequest</a>
to no longer be required.</li>
<li>[Breaking] Changed <code>DaysOfWeek</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#RestartWindow">pipelines.RestartWindow</a>
to type <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#DayOfWeekList">pipelines.DayOfWeekList</a>.</li>
</ul>
<p>OpenAPI SHA: 779817ed8d63031f5ea761fbd25ee84f38feec0d, Date:
2025-01-08</p>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/b83a7262d5dce413535faec96770d3276c270e30"><code>b83a726</code></a>
[Release] Release v0.55.0 (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/1117">#1117</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/23d9c1ea2bf131c426b666573ecac4d798578bb4"><code>23d9c1e</code></a>
[Internal] Move package credentials in config (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/1115">#1115</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/adc94cabf7f3e21aa2b705f3fd799ffb5bf2f7f6"><code>adc94ca</code></a>
[Internal] Decouple serving and oauth2 package (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/1110">#1110</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/83db3cbdab1ee3ccc4ec815d102d43e6c8e06bfe"><code>83db3cb</code></a>
[Internal] Create custom codeql.yml (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/1114">#1114</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/2b55375727620771394ffec0372dfdc3e9e62fdf"><code>2b55375</code></a>
[Internal] Migrate workflows that need write access to use hosted
runners (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/1">#1</a>...</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/03fb2681faf3e6e4ccfaef973763b9eb92463456"><code>03fb268</code></a>
[Internal] Bump x/net, x/crypto dependencies (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/1107">#1107</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/28e1a698ab34e35ca2754372d808c3ce4a700b8a"><code>28e1a69</code></a>
[Internal] Bump staticcheck to 0.5.1 and add go 1.23 test coverage (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/1106">#1106</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/2399d721fe0b67224289330ba47e253482a2bac4"><code>2399d72</code></a>
[Internal] Update Queries test (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/1104">#1104</a>)</li>
<li>See full diff in <a
href="https://github.com/databricks/databricks-sdk-go/compare/v0.54.0...v0.55.0">compare
view</a></li>
</ul>
</details>
<br />

<details>
<summary>Most Recent Ignore Conditions Applied to This Pull
Request</summary>

| Dependency Name | Ignore Conditions |
| --- | --- |
| github.com/databricks/databricks-sdk-go | [>= 0.28.a, < 0.29] |
</details>


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/databricks/databricks-sdk-go&package-manager=go_modules&previous-version=0.54.0&new-version=0.55.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Andrew Nester <andrew.nester@databricks.com>
---
 .codegen/_openapi_sha                         |   2 +-
 .../internal/schema/annotations_openapi.yml   | 266 ++++++++++++++++--
 .../schema/annotations_openapi_overrides.yml  |  38 +++
 bundle/schema/jsonschema.json                 | 123 +++++---
 .../federation-policy/federation-policy.go    |  25 +-
 .../service-principal-federation-policy.go    |  19 +-
 cmd/workspace/jobs/jobs.go                    |  18 +-
 cmd/workspace/pipelines/pipelines.go          |   1 +
 cmd/workspace/shares/shares.go                |   1 +
 go.mod                                        |   2 +-
 go.sum                                        |   4 +-
 11 files changed, 411 insertions(+), 88 deletions(-)

diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha
index 8622b29ca..dfe78790a 100644
--- a/.codegen/_openapi_sha
+++ b/.codegen/_openapi_sha
@@ -1 +1 @@
-a6a317df8327c9b1e5cb59a03a42ffa2aabeef6d
\ No newline at end of file
+779817ed8d63031f5ea761fbd25ee84f38feec0d
\ No newline at end of file
diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml
index e9c893c87..8ff5c9253 100644
--- a/bundle/internal/schema/annotations_openapi.yml
+++ b/bundle/internal/schema/annotations_openapi.yml
@@ -1,4 +1,47 @@
 # This file is auto-generated. DO NOT EDIT.
+github.com/databricks/cli/bundle/config/resources.App:
+  "active_deployment":
+    "description": |-
+      The active deployment of the app. A deployment is considered active when it has been deployed
+      to the app compute.
+  "app_status": {}
+  "compute_status": {}
+  "create_time":
+    "description": |-
+      The creation time of the app. Formatted timestamp in ISO 6801.
+  "creator":
+    "description": |-
+      The email of the user that created the app.
+  "default_source_code_path":
+    "description": |-
+      The default workspace file system path of the source code from which app deployment are
+      created. This field tracks the workspace source code path of the last active deployment.
+  "description":
+    "description": |-
+      The description of the app.
+  "name":
+    "description": |-
+      The name of the app. The name must contain only lowercase alphanumeric characters and hyphens.
+      It must be unique within the workspace.
+  "pending_deployment":
+    "description": |-
+      The pending deployment of the app. A deployment is considered pending when it is being prepared
+      for deployment to the app compute.
+  "resources":
+    "description": |-
+      Resources for the app.
+  "service_principal_client_id": {}
+  "service_principal_id": {}
+  "service_principal_name": {}
+  "update_time":
+    "description": |-
+      The update time of the app. Formatted timestamp in ISO 6801.
+  "updater":
+    "description": |-
+      The email of the user that last updated the app.
+  "url":
+    "description": |-
+      The URL of the app once it is deployed.
 github.com/databricks/cli/bundle/config/resources.Cluster:
   "apply_policy_default_values":
     "description": |-
@@ -220,6 +263,7 @@ github.com/databricks/cli/bundle/config/resources.Job:
   "job_clusters":
     "description": |-
       A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings.
+      If more than 100 job clusters are available, you can paginate through them using :method:jobs/get.
   "max_concurrent_runs":
     "description": |-
       An optional maximum allowed number of concurrent runs of the job.
@@ -250,6 +294,7 @@ github.com/databricks/cli/bundle/config/resources.Job:
   "tasks":
     "description": |-
       A list of task specifications to be executed by this job.
+      If more than 100 tasks are available, you can paginate through them using :method:jobs/get. Use the `next_page_token` field at the object root to determine if more results are available.
   "timeout_seconds":
     "description": |-
       An optional timeout applied to each run of this job. A value of `0` means no timeout.
@@ -489,6 +534,187 @@ github.com/databricks/cli/bundle/config/resources.Volume:
     "description": |-
       The storage location on the cloud
   "volume_type": {}
+github.com/databricks/databricks-sdk-go/service/apps.AppDeployment:
+  "create_time":
+    "description": |-
+      The creation time of the deployment. Formatted timestamp in ISO 6801.
+  "creator":
+    "description": |-
+      The email of the user creates the deployment.
+  "deployment_artifacts":
+    "description": |-
+      The deployment artifacts for an app.
+  "deployment_id":
+    "description": |-
+      The unique id of the deployment.
+  "mode":
+    "description": |-
+      The mode of which the deployment will manage the source code.
+  "source_code_path":
+    "description": |-
+      The workspace file system path of the source code used to create the app deployment. This is different from
+      `deployment_artifacts.source_code_path`, which is the path used by the deployed app. The former refers
+      to the original source code location of the app in the workspace during deployment creation, whereas
+      the latter provides a system generated stable snapshotted source code path used by the deployment.
+  "status":
+    "description": |-
+      Status and status message of the deployment
+  "update_time":
+    "description": |-
+      The update time of the deployment. Formatted timestamp in ISO 6801.
+github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts:
+  "source_code_path":
+    "description": |-
+      The snapshotted workspace file system path of the source code loaded by the deployed app.
+github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentMode:
+  "_":
+    "enum":
+      - |-
+        SNAPSHOT
+      - |-
+        AUTO_SYNC
+github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentState:
+  "_":
+    "enum":
+      - |-
+        SUCCEEDED
+      - |-
+        FAILED
+      - |-
+        IN_PROGRESS
+      - |-
+        CANCELLED
+github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus:
+  "message":
+    "description": |-
+      Message corresponding with the deployment state.
+  "state":
+    "description": |-
+      State of the deployment.
+github.com/databricks/databricks-sdk-go/service/apps.AppResource:
+  "description":
+    "description": |-
+      Description of the App Resource.
+  "job": {}
+  "name":
+    "description": |-
+      Name of the App Resource.
+  "secret": {}
+  "serving_endpoint": {}
+  "sql_warehouse": {}
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob:
+  "id":
+    "description": |-
+      Id of the job to grant permission on.
+  "permission":
+    "description": |-
+      Permissions to grant on the Job. Supported permissions are: "CAN_MANAGE", "IS_OWNER", "CAN_MANAGE_RUN", "CAN_VIEW".
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceJobJobPermission:
+  "_":
+    "enum":
+      - |-
+        CAN_MANAGE
+      - |-
+        IS_OWNER
+      - |-
+        CAN_MANAGE_RUN
+      - |-
+        CAN_VIEW
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret:
+  "key":
+    "description": |-
+      Key of the secret to grant permission on.
+  "permission":
+    "description": |-
+      Permission to grant on the secret scope. For secrets, only one permission is allowed. Permission must be one of: "READ", "WRITE", "MANAGE".
+  "scope":
+    "description": |-
+      Scope of the secret to grant permission on.
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecretSecretPermission:
+  "_":
+    "description": |-
+      Permission to grant on the secret scope. Supported permissions are: "READ", "WRITE", "MANAGE".
+    "enum":
+      - |-
+        READ
+      - |-
+        WRITE
+      - |-
+        MANAGE
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint:
+  "name":
+    "description": |-
+      Name of the serving endpoint to grant permission on.
+  "permission":
+    "description": |-
+      Permission to grant on the serving endpoint. Supported permissions are: "CAN_MANAGE", "CAN_QUERY", "CAN_VIEW".
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpointServingEndpointPermission:
+  "_":
+    "enum":
+      - |-
+        CAN_MANAGE
+      - |-
+        CAN_QUERY
+      - |-
+        CAN_VIEW
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse:
+  "id":
+    "description": |-
+      Id of the SQL warehouse to grant permission on.
+  "permission":
+    "description": |-
+      Permission to grant on the SQL warehouse. Supported permissions are: "CAN_MANAGE", "CAN_USE", "IS_OWNER".
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouseSqlWarehousePermission:
+  "_":
+    "enum":
+      - |-
+        CAN_MANAGE
+      - |-
+        CAN_USE
+      - |-
+        IS_OWNER
+github.com/databricks/databricks-sdk-go/service/apps.ApplicationState:
+  "_":
+    "enum":
+      - |-
+        DEPLOYING
+      - |-
+        RUNNING
+      - |-
+        CRASHED
+      - |-
+        UNAVAILABLE
+github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus:
+  "message":
+    "description": |-
+      Application status message
+  "state":
+    "description": |-
+      State of the application.
+github.com/databricks/databricks-sdk-go/service/apps.ComputeState:
+  "_":
+    "enum":
+      - |-
+        ERROR
+      - |-
+        DELETING
+      - |-
+        STARTING
+      - |-
+        STOPPING
+      - |-
+        UPDATING
+      - |-
+        STOPPED
+      - |-
+        ACTIVE
+github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus:
+  "message":
+    "description": |-
+      Compute status message
+  "state":
+    "description": |-
+      State of the app compute.
 github.com/databricks/databricks-sdk-go/service/catalog.MonitorCronSchedule:
   "pause_status":
     "description": |-
@@ -2116,6 +2342,26 @@ github.com/databricks/databricks-sdk-go/service/ml.ModelVersionTag:
 github.com/databricks/databricks-sdk-go/service/pipelines.CronTrigger:
   "quartz_cron_schedule": {}
   "timezone_id": {}
+github.com/databricks/databricks-sdk-go/service/pipelines.DayOfWeek:
+  "_":
+    "description": |-
+      Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour).
+      If not specified all days of the week will be used.
+    "enum":
+      - |-
+        MONDAY
+      - |-
+        TUESDAY
+      - |-
+        WEDNESDAY
+      - |-
+        THURSDAY
+      - |-
+        FRIDAY
+      - |-
+        SATURDAY
+      - |-
+        SUNDAY
 github.com/databricks/databricks-sdk-go/service/pipelines.DeploymentKind:
   "_":
     "description": |
@@ -2375,26 +2621,6 @@ github.com/databricks/databricks-sdk-go/service/pipelines.RestartWindow:
     "description": |-
       Time zone id of restart window. See https://docs.databricks.com/sql/language-manual/sql-ref-syntax-aux-conf-mgmt-set-timezone.html for details.
       If not specified, UTC will be used.
-github.com/databricks/databricks-sdk-go/service/pipelines.RestartWindowDaysOfWeek:
-  "_":
-    "description": |-
-      Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour).
-      If not specified all days of the week will be used.
-    "enum":
-      - |-
-        MONDAY
-      - |-
-        TUESDAY
-      - |-
-        WEDNESDAY
-      - |-
-        THURSDAY
-      - |-
-        FRIDAY
-      - |-
-        SATURDAY
-      - |-
-        SUNDAY
 github.com/databricks/databricks-sdk-go/service/pipelines.SchemaSpec:
   "destination_catalog":
     "description": |-
diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml
index ef602d6ef..120a12543 100644
--- a/bundle/internal/schema/annotations_openapi_overrides.yml
+++ b/bundle/internal/schema/annotations_openapi_overrides.yml
@@ -1,3 +1,28 @@
+github.com/databricks/cli/bundle/config/resources.App:
+  "app_status":
+    "description": |-
+      PLACEHOLDER
+  "compute_status":
+    "description": |-
+      PLACEHOLDER
+  "config":
+    "description": |-
+      PLACEHOLDER
+  "permissions":
+    "description": |-
+      PLACEHOLDER
+  "service_principal_client_id":
+    "description": |-
+      PLACEHOLDER
+  "service_principal_id":
+    "description": |-
+      PLACEHOLDER
+  "service_principal_name":
+    "description": |-
+      PLACEHOLDER
+  "source_code_path":
+    "description": |-
+      PLACEHOLDER
 github.com/databricks/cli/bundle/config/resources.Cluster:
   "data_security_mode":
     "description": |-
@@ -75,6 +100,19 @@ github.com/databricks/cli/bundle/config/resources.Volume:
   "volume_type":
     "description": |-
       PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppResource:
+  "job":
+    "description": |-
+      PLACEHOLDER
+  "secret":
+    "description": |-
+      PLACEHOLDER
+  "serving_endpoint":
+    "description": |-
+      PLACEHOLDER
+  "sql_warehouse":
+    "description": |-
+      PLACEHOLDER
 github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes:
   "availability":
     "description": |-
diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json
index 81ae1329f..f55813b06 100644
--- a/bundle/schema/jsonschema.json
+++ b/bundle/schema/jsonschema.json
@@ -388,7 +388,7 @@
                         "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthRules"
                       },
                       "job_clusters": {
-                        "description": "A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings.",
+                        "description": "A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings.\nIf more than 100 job clusters are available, you can paginate through them using :method:jobs/get.",
                         "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobCluster"
                       },
                       "max_concurrent_runs": {
@@ -426,7 +426,7 @@
                         "$ref": "#/$defs/map/string"
                       },
                       "tasks": {
-                        "description": "A list of task specifications to be executed by this job.",
+                        "description": "A list of task specifications to be executed by this job.\nIf more than 100 tasks are available, you can paginate through them using :method:jobs/get. Use the `next_page_token` field at the object root to determine if more results are available.",
                         "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.Task"
                       },
                       "timeout_seconds": {
@@ -1662,10 +1662,20 @@
               ]
             },
             "apps.AppDeploymentMode": {
-              "type": "string"
+              "type": "string",
+              "enum": [
+                "SNAPSHOT",
+                "AUTO_SYNC"
+              ]
             },
             "apps.AppDeploymentState": {
-              "type": "string"
+              "type": "string",
+              "enum": [
+                "SUCCEEDED",
+                "FAILED",
+                "IN_PROGRESS",
+                "CANCELLED"
+              ]
             },
             "apps.AppDeploymentStatus": {
               "oneOf": [
@@ -1747,7 +1757,13 @@
               ]
             },
             "apps.AppResourceJobJobPermission": {
-              "type": "string"
+              "type": "string",
+              "enum": [
+                "CAN_MANAGE",
+                "IS_OWNER",
+                "CAN_MANAGE_RUN",
+                "CAN_VIEW"
+              ]
             },
             "apps.AppResourceSecret": {
               "oneOf": [
@@ -1778,7 +1794,13 @@
               ]
             },
             "apps.AppResourceSecretSecretPermission": {
-              "type": "string"
+              "type": "string",
+              "description": "Permission to grant on the secret scope. Supported permissions are: \"READ\", \"WRITE\", \"MANAGE\".",
+              "enum": [
+                "READ",
+                "WRITE",
+                "MANAGE"
+              ]
             },
             "apps.AppResourceServingEndpoint": {
               "oneOf": [
@@ -1805,7 +1827,12 @@
               ]
             },
             "apps.AppResourceServingEndpointServingEndpointPermission": {
-              "type": "string"
+              "type": "string",
+              "enum": [
+                "CAN_MANAGE",
+                "CAN_QUERY",
+                "CAN_VIEW"
+              ]
             },
             "apps.AppResourceSqlWarehouse": {
               "oneOf": [
@@ -1832,10 +1859,21 @@
               ]
             },
             "apps.AppResourceSqlWarehouseSqlWarehousePermission": {
-              "type": "string"
+              "type": "string",
+              "enum": [
+                "CAN_MANAGE",
+                "CAN_USE",
+                "IS_OWNER"
+              ]
             },
             "apps.ApplicationState": {
-              "type": "string"
+              "type": "string",
+              "enum": [
+                "DEPLOYING",
+                "RUNNING",
+                "CRASHED",
+                "UNAVAILABLE"
+              ]
             },
             "apps.ApplicationStatus": {
               "oneOf": [
@@ -1858,7 +1896,16 @@
               ]
             },
             "apps.ComputeState": {
-              "type": "string"
+              "type": "string",
+              "enum": [
+                "ERROR",
+                "DELETING",
+                "STARTING",
+                "STOPPING",
+                "UPDATING",
+                "STOPPED",
+                "ACTIVE"
+              ]
             },
             "apps.ComputeStatus": {
               "oneOf": [
@@ -4569,6 +4616,19 @@
                 }
               ]
             },
+            "pipelines.DayOfWeek": {
+              "type": "string",
+              "description": "Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour).\nIf not specified all days of the week will be used.",
+              "enum": [
+                "MONDAY",
+                "TUESDAY",
+                "WEDNESDAY",
+                "THURSDAY",
+                "FRIDAY",
+                "SATURDAY",
+                "SUNDAY"
+              ]
+            },
             "pipelines.DeploymentKind": {
               "type": "string",
               "description": "The deployment method that manages the pipeline:\n- BUNDLE: The pipeline is managed by a Databricks Asset Bundle.\n",
@@ -5003,7 +5063,7 @@
                   "properties": {
                     "days_of_week": {
                       "description": "Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour).\nIf not specified all days of the week will be used.",
-                      "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/pipelines.RestartWindowDaysOfWeek"
+                      "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/pipelines.DayOfWeek"
                     },
                     "start_hour": {
                       "description": "An integer between 0 and 23 denoting the start hour for the restart window in the 24-hour day.\nContinuous pipeline restart is triggered only within a five-hour window starting at this hour.",
@@ -5025,19 +5085,6 @@
                 }
               ]
             },
-            "pipelines.RestartWindowDaysOfWeek": {
-              "type": "string",
-              "description": "Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour).\nIf not specified all days of the week will be used.",
-              "enum": [
-                "MONDAY",
-                "TUESDAY",
-                "WEDNESDAY",
-                "THURSDAY",
-                "FRIDAY",
-                "SATURDAY",
-                "SUNDAY"
-              ]
-            },
             "pipelines.SchemaSpec": {
               "oneOf": [
                 {
@@ -6619,6 +6666,20 @@
                   }
                 ]
               },
+              "pipelines.DayOfWeek": {
+                "oneOf": [
+                  {
+                    "type": "array",
+                    "items": {
+                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.DayOfWeek"
+                    }
+                  },
+                  {
+                    "type": "string",
+                    "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                  }
+                ]
+              },
               "pipelines.IngestionConfig": {
                 "oneOf": [
                   {
@@ -6675,20 +6736,6 @@
                   }
                 ]
               },
-              "pipelines.RestartWindowDaysOfWeek": {
-                "oneOf": [
-                  {
-                    "type": "array",
-                    "items": {
-                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.RestartWindowDaysOfWeek"
-                    }
-                  },
-                  {
-                    "type": "string",
-                    "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
-                  }
-                ]
-              },
               "serving.AiGatewayRateLimit": {
                 "oneOf": [
                   {
diff --git a/cmd/account/federation-policy/federation-policy.go b/cmd/account/federation-policy/federation-policy.go
index d78ac709a..897532e3a 100755
--- a/cmd/account/federation-policy/federation-policy.go
+++ b/cmd/account/federation-policy/federation-policy.go
@@ -111,7 +111,7 @@ func newCreate() *cobra.Command {
 	cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
 	cmd.Flags().StringVar(&createReq.Policy.Description, "description", createReq.Policy.Description, `Description of the federation policy.`)
-	cmd.Flags().StringVar(&createReq.Policy.Name, "name", createReq.Policy.Name, `Name of the federation policy.`)
+	cmd.Flags().StringVar(&createReq.Policy.Name, "name", createReq.Policy.Name, `Resource name for the federation policy.`)
 	// TODO: complex arg: oidc_policy
 
 	cmd.Use = "create"
@@ -180,7 +180,10 @@ func newDelete() *cobra.Command {
 
 	cmd.Use = "delete POLICY_ID"
 	cmd.Short = `Delete account federation policy.`
-	cmd.Long = `Delete account federation policy.`
+	cmd.Long = `Delete account federation policy.
+
+  Arguments:
+    POLICY_ID: The identifier for the federation policy.`
 
 	cmd.Annotations = make(map[string]string)
 
@@ -233,7 +236,10 @@ func newGet() *cobra.Command {
 
 	cmd.Use = "get POLICY_ID"
 	cmd.Short = `Get account federation policy.`
-	cmd.Long = `Get account federation policy.`
+	cmd.Long = `Get account federation policy.
+
+  Arguments:
+    POLICY_ID: The identifier for the federation policy.`
 
 	cmd.Annotations = make(map[string]string)
 
@@ -339,24 +345,20 @@ func newUpdate() *cobra.Command {
 	cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
 	cmd.Flags().StringVar(&updateReq.Policy.Description, "description", updateReq.Policy.Description, `Description of the federation policy.`)
-	cmd.Flags().StringVar(&updateReq.Policy.Name, "name", updateReq.Policy.Name, `Name of the federation policy.`)
+	cmd.Flags().StringVar(&updateReq.Policy.Name, "name", updateReq.Policy.Name, `Resource name for the federation policy.`)
 	// TODO: complex arg: oidc_policy
 
-	cmd.Use = "update POLICY_ID UPDATE_MASK"
+	cmd.Use = "update POLICY_ID"
 	cmd.Short = `Update account federation policy.`
 	cmd.Long = `Update account federation policy.
 
   Arguments:
-    POLICY_ID: 
-    UPDATE_MASK: Field mask is required to be passed into the PATCH request. Field mask
-      specifies which fields of the setting payload will be updated. The field
-      mask needs to be supplied as single string. To specify multiple fields in
-      the field mask, use comma as the separator (no space).`
+    POLICY_ID: The identifier for the federation policy.`
 
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
-		check := root.ExactArgs(2)
+		check := root.ExactArgs(1)
 		return check(cmd, args)
 	}
 
@@ -378,7 +380,6 @@ func newUpdate() *cobra.Command {
 			}
 		}
 		updateReq.PolicyId = args[0]
-		updateReq.UpdateMask = args[1]
 
 		response, err := a.FederationPolicy.Update(ctx, updateReq)
 		if err != nil {
diff --git a/cmd/account/service-principal-federation-policy/service-principal-federation-policy.go b/cmd/account/service-principal-federation-policy/service-principal-federation-policy.go
index 77f73bcd0..5f33505a1 100755
--- a/cmd/account/service-principal-federation-policy/service-principal-federation-policy.go
+++ b/cmd/account/service-principal-federation-policy/service-principal-federation-policy.go
@@ -118,7 +118,7 @@ func newCreate() *cobra.Command {
 	cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
 	cmd.Flags().StringVar(&createReq.Policy.Description, "description", createReq.Policy.Description, `Description of the federation policy.`)
-	cmd.Flags().StringVar(&createReq.Policy.Name, "name", createReq.Policy.Name, `Name of the federation policy.`)
+	cmd.Flags().StringVar(&createReq.Policy.Name, "name", createReq.Policy.Name, `Resource name for the federation policy.`)
 	// TODO: complex arg: oidc_policy
 
 	cmd.Use = "create SERVICE_PRINCIPAL_ID"
@@ -198,7 +198,7 @@ func newDelete() *cobra.Command {
 
   Arguments:
     SERVICE_PRINCIPAL_ID: The service principal id for the federation policy.
-    POLICY_ID: `
+    POLICY_ID: The identifier for the federation policy.`
 
 	cmd.Annotations = make(map[string]string)
 
@@ -259,7 +259,7 @@ func newGet() *cobra.Command {
 
   Arguments:
     SERVICE_PRINCIPAL_ID: The service principal id for the federation policy.
-    POLICY_ID: `
+    POLICY_ID: The identifier for the federation policy.`
 
 	cmd.Annotations = make(map[string]string)
 
@@ -377,25 +377,21 @@ func newUpdate() *cobra.Command {
 	cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
 	cmd.Flags().StringVar(&updateReq.Policy.Description, "description", updateReq.Policy.Description, `Description of the federation policy.`)
-	cmd.Flags().StringVar(&updateReq.Policy.Name, "name", updateReq.Policy.Name, `Name of the federation policy.`)
+	cmd.Flags().StringVar(&updateReq.Policy.Name, "name", updateReq.Policy.Name, `Resource name for the federation policy.`)
 	// TODO: complex arg: oidc_policy
 
-	cmd.Use = "update SERVICE_PRINCIPAL_ID POLICY_ID UPDATE_MASK"
+	cmd.Use = "update SERVICE_PRINCIPAL_ID POLICY_ID"
 	cmd.Short = `Update service principal federation policy.`
 	cmd.Long = `Update service principal federation policy.
 
   Arguments:
     SERVICE_PRINCIPAL_ID: The service principal id for the federation policy.
-    POLICY_ID: 
-    UPDATE_MASK: Field mask is required to be passed into the PATCH request. Field mask
-      specifies which fields of the setting payload will be updated. The field
-      mask needs to be supplied as single string. To specify multiple fields in
-      the field mask, use comma as the separator (no space).`
+    POLICY_ID: The identifier for the federation policy.`
 
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
-		check := root.ExactArgs(3)
+		check := root.ExactArgs(2)
 		return check(cmd, args)
 	}
 
@@ -421,7 +417,6 @@ func newUpdate() *cobra.Command {
 			return fmt.Errorf("invalid SERVICE_PRINCIPAL_ID: %s", args[0])
 		}
 		updateReq.PolicyId = args[1]
-		updateReq.UpdateMask = args[2]
 
 		response, err := a.ServicePrincipalFederationPolicy.Update(ctx, updateReq)
 		if err != nil {
diff --git a/cmd/workspace/jobs/jobs.go b/cmd/workspace/jobs/jobs.go
index b067937e2..38a88f014 100755
--- a/cmd/workspace/jobs/jobs.go
+++ b/cmd/workspace/jobs/jobs.go
@@ -625,11 +625,19 @@ func newGet() *cobra.Command {
 
 	// TODO: short flags
 
+	cmd.Flags().StringVar(&getReq.PageToken, "page-token", getReq.PageToken, `Use next_page_token returned from the previous GetJob to request the next page of the job's sub-resources.`)
+
 	cmd.Use = "get JOB_ID"
 	cmd.Short = `Get a single job.`
 	cmd.Long = `Get a single job.
   
   Retrieves the details for a single job.
+  
+  In Jobs API 2.2, requests for a single job support pagination of tasks and
+  job_clusters when either exceeds 100 elements. Use the next_page_token
+  field to check for more results and pass its value as the page_token in
+  subsequent requests. Arrays with fewer than 100 elements in a page will be
+  empty on later pages.
 
   Arguments:
     JOB_ID: The canonical identifier of the job to retrieve information about. This
@@ -847,13 +855,19 @@ func newGetRun() *cobra.Command {
 
 	cmd.Flags().BoolVar(&getRunReq.IncludeHistory, "include-history", getRunReq.IncludeHistory, `Whether to include the repair history in the response.`)
 	cmd.Flags().BoolVar(&getRunReq.IncludeResolvedValues, "include-resolved-values", getRunReq.IncludeResolvedValues, `Whether to include resolved parameter values in the response.`)
-	cmd.Flags().StringVar(&getRunReq.PageToken, "page-token", getRunReq.PageToken, `To list the next page of job tasks, set this field to the value of the next_page_token returned in the GetJob response.`)
+	cmd.Flags().StringVar(&getRunReq.PageToken, "page-token", getRunReq.PageToken, `Use next_page_token returned from the previous GetRun to request the next page of the run's sub-resources.`)
 
 	cmd.Use = "get-run RUN_ID"
 	cmd.Short = `Get a single job run.`
 	cmd.Long = `Get a single job run.
   
-  Retrieve the metadata of a run.
+  Retrieves the metadata of a run.
+  
+  In Jobs API 2.2, requests for a single job run support pagination of tasks
+  and job_clusters when either exceeds 100 elements. Use the next_page_token
+  field to check for more results and pass its value as the page_token in
+  subsequent requests. Arrays with fewer than 100 elements in a page will be
+  empty on later pages.
 
   Arguments:
     RUN_ID: The canonical identifier of the run for which to retrieve the metadata.
diff --git a/cmd/workspace/pipelines/pipelines.go b/cmd/workspace/pipelines/pipelines.go
index 38636e83b..e94d4c5a8 100755
--- a/cmd/workspace/pipelines/pipelines.go
+++ b/cmd/workspace/pipelines/pipelines.go
@@ -974,6 +974,7 @@ func newUpdate() *cobra.Command {
 	cmd.Flags().BoolVar(&updateReq.Photon, "photon", updateReq.Photon, `Whether Photon is enabled for this pipeline.`)
 	cmd.Flags().StringVar(&updateReq.PipelineId, "pipeline-id", updateReq.PipelineId, `Unique identifier for this pipeline.`)
 	// TODO: complex arg: restart_window
+	// TODO: complex arg: run_as
 	cmd.Flags().StringVar(&updateReq.Schema, "schema", updateReq.Schema, `The default schema (database) where tables are read from or published to.`)
 	cmd.Flags().BoolVar(&updateReq.Serverless, "serverless", updateReq.Serverless, `Whether serverless compute is enabled for this pipeline.`)
 	cmd.Flags().StringVar(&updateReq.Storage, "storage", updateReq.Storage, `DBFS root directory for storing checkpoints and tables.`)
diff --git a/cmd/workspace/shares/shares.go b/cmd/workspace/shares/shares.go
index f70963f29..62c3407f4 100755
--- a/cmd/workspace/shares/shares.go
+++ b/cmd/workspace/shares/shares.go
@@ -391,6 +391,7 @@ func newUpdate() *cobra.Command {
 
 	cmd.Flags().StringVar(&updateReq.Comment, "comment", updateReq.Comment, `User-provided free-form text description.`)
 	cmd.Flags().StringVar(&updateReq.NewName, "new-name", updateReq.NewName, `New name for the share.`)
+	cmd.Flags().StringVar(&updateReq.Owner, "owner", updateReq.Owner, `Username of current owner of share.`)
 	cmd.Flags().StringVar(&updateReq.StorageRoot, "storage-root", updateReq.StorageRoot, `Storage root URL for the share.`)
 	// TODO: array: updates
 
diff --git a/go.mod b/go.mod
index ed2ff12ad..0ef800d7b 100644
--- a/go.mod
+++ b/go.mod
@@ -7,7 +7,7 @@ toolchain go1.23.4
 require (
 	github.com/Masterminds/semver/v3 v3.3.1 // MIT
 	github.com/briandowns/spinner v1.23.1 // Apache 2.0
-	github.com/databricks/databricks-sdk-go v0.54.0 // Apache 2.0
+	github.com/databricks/databricks-sdk-go v0.55.0 // Apache 2.0
 	github.com/fatih/color v1.18.0 // MIT
 	github.com/google/uuid v1.6.0 // BSD-3-Clause
 	github.com/hashicorp/go-version v1.7.0 // MPL 2.0
diff --git a/go.sum b/go.sum
index 2b9290b71..b1364cb26 100644
--- a/go.sum
+++ b/go.sum
@@ -32,8 +32,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cyphar/filepath-securejoin v0.2.5 h1:6iR5tXJ/e6tJZzzdMc1km3Sa7RRIVBKAK32O2s7AYfo=
 github.com/cyphar/filepath-securejoin v0.2.5/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
-github.com/databricks/databricks-sdk-go v0.54.0 h1:L8gsA3NXs+uYU3QtW/OUgjxMQxOH24k0MT9JhB3zLlM=
-github.com/databricks/databricks-sdk-go v0.54.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU=
+github.com/databricks/databricks-sdk-go v0.55.0 h1:ReziD6spzTDltM0ml80LggKo27F3oUjgTinCFDJDnak=
+github.com/databricks/databricks-sdk-go v0.55.0/go.mod h1:JpLizplEs+up9/Z4Xf2x++o3sM9eTTWFGzIXAptKJzI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

From 82e35530b02d1b2246a012ef3a1720c3791acb16 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Tue, 14 Jan 2025 19:23:34 +0100
Subject: [PATCH 074/247] Add acceptance tests for builtin templates (#2135)

## Changes

To accommodate:
* Add the server URL to the set of output replacements
* Include a call to the permissions API to the dummy server
* Run the main script in a subshell to isolate working directory changes
---
 acceptance/acceptance_test.go                 |  7 +++-
 acceptance/bundle/init/dbt-sql/input.json     |  6 ++++
 acceptance/bundle/init/dbt-sql/output.txt     | 32 +++++++++++++++++++
 acceptance/bundle/init/dbt-sql/script         |  5 +++
 acceptance/bundle/init/dbt-sql/script.cleanup |  1 +
 .../bundle/init/default-python/input.json     |  6 ++++
 .../bundle/init/default-python/output.txt     | 30 +++++++++++++++++
 acceptance/bundle/init/default-python/script  |  5 +++
 .../bundle/init/default-python/script.cleanup |  1 +
 acceptance/bundle/init/default-sql/input.json |  6 ++++
 acceptance/bundle/init/default-sql/output.txt | 32 +++++++++++++++++++
 acceptance/bundle/init/default-sql/script     |  5 +++
 .../bundle/init/default-sql/script.cleanup    |  1 +
 acceptance/server_test.go                     | 30 ++++++++++++++---
 14 files changed, 161 insertions(+), 6 deletions(-)
 create mode 100644 acceptance/bundle/init/dbt-sql/input.json
 create mode 100644 acceptance/bundle/init/dbt-sql/output.txt
 create mode 100644 acceptance/bundle/init/dbt-sql/script
 create mode 100644 acceptance/bundle/init/dbt-sql/script.cleanup
 create mode 100644 acceptance/bundle/init/default-python/input.json
 create mode 100644 acceptance/bundle/init/default-python/output.txt
 create mode 100644 acceptance/bundle/init/default-python/script
 create mode 100644 acceptance/bundle/init/default-python/script.cleanup
 create mode 100644 acceptance/bundle/init/default-sql/input.json
 create mode 100644 acceptance/bundle/init/default-sql/output.txt
 create mode 100644 acceptance/bundle/init/default-sql/script
 create mode 100644 acceptance/bundle/init/default-sql/script.cleanup

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 4b5146503..62c56a325 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -66,7 +66,7 @@ func TestAccept(t *testing.T) {
 		server := StartServer(t)
 		AddHandlers(server)
 		// Redirect API access to local server:
-		t.Setenv("DATABRICKS_HOST", fmt.Sprintf("http://127.0.0.1:%d", server.Port))
+		t.Setenv("DATABRICKS_HOST", server.URL)
 		t.Setenv("DATABRICKS_TOKEN", "dapi1234")
 
 		homeDir := t.TempDir()
@@ -217,6 +217,11 @@ func doComparison(t *testing.T, pathExpected, pathNew, valueNew string) {
 // Note, cleanups are not executed if main script fails; that's not a huge issue, since it runs it temp dir.
 func readMergedScriptContents(t *testing.T, dir string) string {
 	scriptContents := testutil.ReadFile(t, filepath.Join(dir, EntryPointScript))
+
+	// Wrap script contents in a subshell such that changing the working
+	// directory only affects the main script and not cleanup.
+	scriptContents = "(\n" + scriptContents + ")\n"
+
 	prepares := []string{}
 	cleanups := []string{}
 
diff --git a/acceptance/bundle/init/dbt-sql/input.json b/acceptance/bundle/init/dbt-sql/input.json
new file mode 100644
index 000000000..201ac9667
--- /dev/null
+++ b/acceptance/bundle/init/dbt-sql/input.json
@@ -0,0 +1,6 @@
+{
+    "project_name": "my_dbt_sql",
+    "http_path": "/sql/2.0/warehouses/f00dcafe",
+    "default_catalog": "main",
+    "personal_schemas": "yes, use a schema based on the current user name during development"
+}
diff --git a/acceptance/bundle/init/dbt-sql/output.txt b/acceptance/bundle/init/dbt-sql/output.txt
new file mode 100644
index 000000000..384013d14
--- /dev/null
+++ b/acceptance/bundle/init/dbt-sql/output.txt
@@ -0,0 +1,32 @@
+
+>>> $CLI bundle init dbt-sql --config-file ./input.json
+
+Welcome to the dbt template for Databricks Asset Bundles!
+
+A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html.
+workspace_host: http://$DATABRICKS_HOST
+
+📊 Your new project has been created in the 'my_dbt_sql' directory!
+If you already have dbt installed, just type 'cd my_dbt_sql; dbt init' to get started.
+Refer to the README.md file for full "getting started" guide and production setup instructions.
+
+
+>>> $CLI bundle validate -t dev
+Name: my_dbt_sql
+Target: dev
+Workspace:
+  Host: http://$DATABRICKS_HOST
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/my_dbt_sql/dev
+
+Validation OK!
+
+>>> $CLI bundle validate -t prod
+Name: my_dbt_sql
+Target: prod
+Workspace:
+  Host: http://$DATABRICKS_HOST
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/my_dbt_sql/prod
+
+Validation OK!
diff --git a/acceptance/bundle/init/dbt-sql/script b/acceptance/bundle/init/dbt-sql/script
new file mode 100644
index 000000000..5f6efda99
--- /dev/null
+++ b/acceptance/bundle/init/dbt-sql/script
@@ -0,0 +1,5 @@
+trace $CLI bundle init dbt-sql --config-file ./input.json
+
+cd my_dbt_sql
+trace $CLI bundle validate -t dev
+trace $CLI bundle validate -t prod
diff --git a/acceptance/bundle/init/dbt-sql/script.cleanup b/acceptance/bundle/init/dbt-sql/script.cleanup
new file mode 100644
index 000000000..b717c8d4a
--- /dev/null
+++ b/acceptance/bundle/init/dbt-sql/script.cleanup
@@ -0,0 +1 @@
+rm -fr my_dbt_sql
diff --git a/acceptance/bundle/init/default-python/input.json b/acceptance/bundle/init/default-python/input.json
new file mode 100644
index 000000000..3e1d79c68
--- /dev/null
+++ b/acceptance/bundle/init/default-python/input.json
@@ -0,0 +1,6 @@
+{
+    "project_name": "my_default_python",
+    "include_notebook": "yes",
+    "include_dlt": "yes",
+    "include_python": "yes"
+}
diff --git a/acceptance/bundle/init/default-python/output.txt b/acceptance/bundle/init/default-python/output.txt
new file mode 100644
index 000000000..c389ef0ad
--- /dev/null
+++ b/acceptance/bundle/init/default-python/output.txt
@@ -0,0 +1,30 @@
+
+>>> $CLI bundle init default-python --config-file ./input.json
+
+Welcome to the default Python template for Databricks Asset Bundles!
+Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): http://$DATABRICKS_HOST
+
+✨ Your new project has been created in the 'my_default_python' directory!
+
+Please refer to the README.md file for "getting started" instructions.
+See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.
+
+>>> $CLI bundle validate -t dev
+Name: my_default_python
+Target: dev
+Workspace:
+  Host: http://$DATABRICKS_HOST
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/my_default_python/dev
+
+Validation OK!
+
+>>> $CLI bundle validate -t prod
+Name: my_default_python
+Target: prod
+Workspace:
+  Host: http://$DATABRICKS_HOST
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/my_default_python/prod
+
+Validation OK!
diff --git a/acceptance/bundle/init/default-python/script b/acceptance/bundle/init/default-python/script
new file mode 100644
index 000000000..84834ce3f
--- /dev/null
+++ b/acceptance/bundle/init/default-python/script
@@ -0,0 +1,5 @@
+trace $CLI bundle init default-python --config-file ./input.json
+
+cd my_default_python
+trace $CLI bundle validate -t dev
+trace $CLI bundle validate -t prod
diff --git a/acceptance/bundle/init/default-python/script.cleanup b/acceptance/bundle/init/default-python/script.cleanup
new file mode 100644
index 000000000..4fd2e4aa7
--- /dev/null
+++ b/acceptance/bundle/init/default-python/script.cleanup
@@ -0,0 +1 @@
+rm -fr my_default_python
diff --git a/acceptance/bundle/init/default-sql/input.json b/acceptance/bundle/init/default-sql/input.json
new file mode 100644
index 000000000..c728d25de
--- /dev/null
+++ b/acceptance/bundle/init/default-sql/input.json
@@ -0,0 +1,6 @@
+{
+    "project_name": "my_default_sql",
+    "http_path": "/sql/2.0/warehouses/f00dcafe",
+    "default_catalog": "main",
+    "personal_schemas": "yes, automatically use a schema based on the current user name during development"
+}
diff --git a/acceptance/bundle/init/default-sql/output.txt b/acceptance/bundle/init/default-sql/output.txt
new file mode 100644
index 000000000..3605206dc
--- /dev/null
+++ b/acceptance/bundle/init/default-sql/output.txt
@@ -0,0 +1,32 @@
+
+>>> $CLI bundle init default-sql --config-file ./input.json
+
+Welcome to the default SQL template for Databricks Asset Bundles!
+
+A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html.
+workspace_host: http://$DATABRICKS_HOST
+
+✨ Your new project has been created in the 'my_default_sql' directory!
+
+Please refer to the README.md file for "getting started" instructions.
+See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.
+
+>>> $CLI bundle validate -t dev
+Name: my_default_sql
+Target: dev
+Workspace:
+  Host: http://$DATABRICKS_HOST
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/my_default_sql/dev
+
+Validation OK!
+
+>>> $CLI bundle validate -t prod
+Name: my_default_sql
+Target: prod
+Workspace:
+  Host: http://$DATABRICKS_HOST
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/my_default_sql/prod
+
+Validation OK!
diff --git a/acceptance/bundle/init/default-sql/script b/acceptance/bundle/init/default-sql/script
new file mode 100644
index 000000000..6f7332a39
--- /dev/null
+++ b/acceptance/bundle/init/default-sql/script
@@ -0,0 +1,5 @@
+trace $CLI bundle init default-sql --config-file ./input.json
+
+cd my_default_sql
+trace $CLI bundle validate -t dev
+trace $CLI bundle validate -t prod
diff --git a/acceptance/bundle/init/default-sql/script.cleanup b/acceptance/bundle/init/default-sql/script.cleanup
new file mode 100644
index 000000000..c46d9ad60
--- /dev/null
+++ b/acceptance/bundle/init/default-sql/script.cleanup
@@ -0,0 +1 @@
+rm -fr my_default_sql
diff --git a/acceptance/server_test.go b/acceptance/server_test.go
index 7b21e198f..0d10fbea1 100644
--- a/acceptance/server_test.go
+++ b/acceptance/server_test.go
@@ -2,11 +2,11 @@ package acceptance_test
 
 import (
 	"encoding/json"
-	"net"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
+	"github.com/databricks/databricks-sdk-go/service/catalog"
 	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/iam"
 	"github.com/databricks/databricks-sdk-go/service/workspace"
@@ -14,8 +14,7 @@ import (
 
 type TestServer struct {
 	*httptest.Server
-	Mux  *http.ServeMux
-	Port int
+	Mux *http.ServeMux
 }
 
 type HandlerFunc func(r *http.Request) (any, error)
@@ -23,12 +22,10 @@ type HandlerFunc func(r *http.Request) (any, error)
 func NewTestServer() *TestServer {
 	mux := http.NewServeMux()
 	server := httptest.NewServer(mux)
-	port := server.Listener.Addr().(*net.TCPAddr).Port
 
 	return &TestServer{
 		Server: server,
 		Mux:    mux,
-		Port:   port,
 	}
 }
 
@@ -126,4 +123,27 @@ func AddHandlers(server *TestServer) {
 			ResourceId: "1001",
 		}, nil
 	})
+
+	server.Handle("/api/2.1/unity-catalog/current-metastore-assignment", func(r *http.Request) (any, error) {
+		return catalog.MetastoreAssignment{
+			DefaultCatalogName: "main",
+		}, nil
+	})
+
+	server.Handle("/api/2.0/permissions/directories/1001", func(r *http.Request) (any, error) {
+		return workspace.WorkspaceObjectPermissions{
+			ObjectId:   "1001",
+			ObjectType: "DIRECTORY",
+			AccessControlList: []workspace.WorkspaceObjectAccessControlResponse{
+				{
+					UserName: "tester@databricks.com",
+					AllPermissions: []workspace.WorkspaceObjectPermission{
+						{
+							PermissionLevel: "CAN_MANAGE",
+						},
+					},
+				},
+			},
+		}, nil
+	})
 }

From 6a7eefa54be1d0e11e4290e7ab2d3ecd50fed2e9 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 15 Jan 2025 09:43:40 +0100
Subject: [PATCH 075/247] Use the same test names on Win as on other OSes
 (#2149)

## Changes
Use name format "TestAccept/bundle/variables/host" (previously slashes
were reversed on Windows).

## Tests
Manually run "go test ./acceptance -v -run
TestAccept/bundle/variables/host" in Windows VM.
---
 acceptance/acceptance_test.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 62c56a325..529077b05 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -87,7 +87,8 @@ func TestAccept(t *testing.T) {
 	require.NotEmpty(t, testDirs)
 
 	for _, dir := range testDirs {
-		t.Run(dir, func(t *testing.T) {
+		testName := strings.ReplaceAll(dir, "\\", "/")
+		t.Run(testName, func(t *testing.T) {
 			t.Parallel()
 			runTest(t, dir, coverDir, repls)
 		})

From cc44e368b8822a2b4a10b5733c6fad713596cff0 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 15 Jan 2025 09:46:06 +0100
Subject: [PATCH 076/247] Golden files: always include JSON-ed string (cont)
 (#2147)

## Changes
Follow up to #2142 which did not actually enabled JSON conversion
because of reversed condition on err.

## Tests
Tested via #2118
---
 libs/testdiff/golden.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libs/testdiff/golden.go b/libs/testdiff/golden.go
index 78973b468..dd07df408 100644
--- a/libs/testdiff/golden.go
+++ b/libs/testdiff/golden.go
@@ -129,9 +129,9 @@ func (r *ReplacementsContext) Set(old, new string) {
 	// In that case we cannot rely that json(old) == '"{old}"' and need to add it explicitly.
 
 	encodedNew, err := json.Marshal(new)
-	if err != nil {
+	if err == nil {
 		encodedOld, err := json.Marshal(old)
-		if err != nil {
+		if err == nil {
 			r.Repls = append(r.Repls, Replacement{Old: string(encodedOld), New: string(encodedNew)})
 		}
 	}

From 55494a0bdaa58ed7094d929d9fd0dd1b02f91613 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 15 Jan 2025 10:34:51 +0100
Subject: [PATCH 077/247] Add test about using variable in bundle.git.branch
 (#2118)

This test checks load git details functionality + variable interpolation
there.

The variables are not working there because LoadGitDetails mutator is
running before variable interpolation.

Additionally, correctly replace tmp path that is used for DATABRICKS_TF_EXEC_PATH
---
 acceptance/acceptance_test.go                 |  7 ++
 .../variables/git-branch/databricks.yml       | 19 ++++
 .../bundle/variables/git-branch/output.txt    | 98 +++++++++++++++++++
 acceptance/bundle/variables/git-branch/script |  6 ++
 acceptance/script.prepare                     | 12 ++-
 5 files changed, 139 insertions(+), 3 deletions(-)
 create mode 100644 acceptance/bundle/variables/git-branch/databricks.yml
 create mode 100644 acceptance/bundle/variables/git-branch/output.txt
 create mode 100644 acceptance/bundle/variables/git-branch/script

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 529077b05..c7f40d11e 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -59,6 +59,13 @@ func TestAccept(t *testing.T) {
 	repls := testdiff.ReplacementsContext{}
 	repls.Set(execPath, "$CLI")
 
+	tempHomeDir := t.TempDir()
+	repls.Set(tempHomeDir, "$TMPHOME")
+	t.Logf("$TMPHOME=%v", tempHomeDir)
+
+	// Prevent CLI from downloading terraform in each test:
+	t.Setenv("DATABRICKS_TF_EXEC_PATH", tempHomeDir)
+
 	ctx := context.Background()
 	cloudEnv := os.Getenv("CLOUD_ENV")
 
diff --git a/acceptance/bundle/variables/git-branch/databricks.yml b/acceptance/bundle/variables/git-branch/databricks.yml
new file mode 100644
index 000000000..7cf210722
--- /dev/null
+++ b/acceptance/bundle/variables/git-branch/databricks.yml
@@ -0,0 +1,19 @@
+bundle:
+  name: git
+  git:
+    # This is currently not supported
+    branch: ${var.deployment_branch}
+
+variables:
+  deployment_branch:
+    # By setting deployment_branch to "" we set bundle.git.branch to "" which is the same unsetting it.
+    # This this should make CLI read branch from git and update bundle.git.branch accordingly. It should
+    # Also set bundle.git.inferred to true.
+    default: ""
+
+targets:
+  prod:
+    default: true
+  dev:
+    variables:
+      deployment_branch: dev-branch
diff --git a/acceptance/bundle/variables/git-branch/output.txt b/acceptance/bundle/variables/git-branch/output.txt
new file mode 100644
index 000000000..d6d824394
--- /dev/null
+++ b/acceptance/bundle/variables/git-branch/output.txt
@@ -0,0 +1,98 @@
+
+>>> $CLI bundle validate -o json
+{
+  "bundle": {
+    "environment": "prod",
+    "git": {
+      "actual_branch": "main",
+      "branch": "",
+      "bundle_root_path": ".",
+    },
+    "name": "git",
+    "target": "prod",
+    "terraform": {
+      "exec_path": "$TMPHOME"
+    }
+  },
+  "sync": {
+    "paths": [
+      "."
+    ]
+  },
+  "targets": null,
+  "variables": {
+    "deployment_branch": {
+      "default": "",
+      "value": ""
+    }
+  },
+  "workspace": {
+    "artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/artifacts",
+    "current_user": {
+      "short_name": "$USERNAME",
+      "userName": "$USERNAME"
+    },
+    "file_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/files",
+    "resource_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/resources",
+    "root_path": "/Workspace/Users/$USERNAME/.bundle/git/prod",
+    "state_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/state"
+  }
+}
+
+>>> $CLI bundle validate
+Name: git
+Target: prod
+Workspace:
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/git/prod
+
+Validation OK!
+
+>>> $CLI bundle validate -o json -t dev
+{
+  "bundle": {
+    "environment": "dev",
+    "git": {
+      "actual_branch": "main",
+      "branch": "dev-branch",
+      "bundle_root_path": ".",
+    },
+    "name": "git",
+    "target": "dev",
+    "terraform": {
+      "exec_path": "$TMPHOME"
+    }
+  },
+  "sync": {
+    "paths": [
+      "."
+    ]
+  },
+  "targets": null,
+  "variables": {
+    "deployment_branch": {
+      "default": "dev-branch",
+      "value": "dev-branch"
+    }
+  },
+  "workspace": {
+    "artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/artifacts",
+    "current_user": {
+      "short_name": "$USERNAME",
+      "userName": "$USERNAME"
+    },
+    "file_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/files",
+    "resource_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/resources",
+    "root_path": "/Workspace/Users/$USERNAME/.bundle/git/dev",
+    "state_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/state"
+  }
+}
+
+>>> $CLI bundle validate -t dev
+Name: git
+Target: dev
+Workspace:
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/git/dev
+
+Validation OK!
diff --git a/acceptance/bundle/variables/git-branch/script b/acceptance/bundle/variables/git-branch/script
new file mode 100644
index 000000000..aed881f1f
--- /dev/null
+++ b/acceptance/bundle/variables/git-branch/script
@@ -0,0 +1,6 @@
+git-repo-init
+trace $CLI bundle validate -o json | grep -v '"commit"'
+trace $CLI bundle validate
+trace $CLI bundle validate -o json -t dev | grep -v '"commit"'
+trace $CLI bundle validate -t dev | grep -v '"commit"'
+rm -fr .git
diff --git a/acceptance/script.prepare b/acceptance/script.prepare
index 3f1bb2acc..61061b59e 100644
--- a/acceptance/script.prepare
+++ b/acceptance/script.prepare
@@ -1,6 +1,3 @@
-# Prevent CLI from downloading terraform in each test:
-export DATABRICKS_TF_EXEC_PATH=/tmp/
-
 errcode() {
     # Temporarily disable 'set -e' to prevent the script from exiting on error
     set +e
@@ -34,3 +31,12 @@ trace() {
 
     return $?
 }
+
+git-repo-init() {
+    git init -qb main
+    git config --global core.autocrlf false
+    git config user.name "Tester"
+    git config user.email "tester@databricks.com"
+    git add databricks.yml
+    git commit -qm 'Add databricks.yml'
+}

From 25f8ee8d6631d7f76bec4e2d2609d825fa5eee31 Mon Sep 17 00:00:00 2001
From: Gleb Kanterov <kanterov@users.noreply.github.com>
Date: Wed, 15 Jan 2025 10:40:29 +0100
Subject: [PATCH 078/247] Format default-python template (#2110)

## Changes
Format code in default-python template, so it's already pre-formatted.

## Tests

```
$ databricks bundle init libs/template/templates/default-python
$ ruff format --diff my_project
6 files already formatted
```
---
 .../scratch/exploration.ipynb.tmpl            |  3 ++-
 .../template/{{.project_name}}/setup.py.tmpl  | 16 +++++++++-----
 .../src/dlt_pipeline.ipynb.tmpl               |  8 ++++---
 .../src/{{.project_name}}/main.py.tmpl        | 22 +++++++++++--------
 4 files changed, 30 insertions(+), 19 deletions(-)

diff --git a/libs/template/templates/default-python/template/{{.project_name}}/scratch/exploration.ipynb.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/scratch/exploration.ipynb.tmpl
index 42164dff0..d3e9beef3 100644
--- a/libs/template/templates/default-python/template/{{.project_name}}/scratch/exploration.ipynb.tmpl
+++ b/libs/template/templates/default-python/template/{{.project_name}}/scratch/exploration.ipynb.tmpl
@@ -29,7 +29,8 @@
    "source": [
    {{- if (eq .include_python "yes") }}
     "import sys\n",
-    "sys.path.append('../src')\n",
+    "\n",
+    "sys.path.append(\"../src\")\n",
     "from {{.project_name}} import main\n",
     "\n",
     "main.get_taxis(spark).show(10)"
diff --git a/libs/template/templates/default-python/template/{{.project_name}}/setup.py.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/setup.py.tmpl
index a0852c725..e3b70c605 100644
--- a/libs/template/templates/default-python/template/{{.project_name}}/setup.py.tmpl
+++ b/libs/template/templates/default-python/template/{{.project_name}}/setup.py.tmpl
@@ -5,28 +5,32 @@ This file is primarily used by the setuptools library and typically should not
 be executed directly. See README.md for how to deploy, test, and run
 the {{.project_name}} project.
 """
+
 from setuptools import setup, find_packages
 
 import sys
-sys.path.append('./src')
+
+sys.path.append("./src")
 
 import datetime
 import {{.project_name}}
 
+local_version = datetime.datetime.utcnow().strftime("%Y%m%d.%H%M%S")
+
 setup(
     name="{{.project_name}}",
     # We use timestamp as Local version identifier (https://peps.python.org/pep-0440/#local-version-identifiers.)
     # to ensure that changes to wheel package are picked up when used on all-purpose clusters
-    version={{.project_name}}.__version__ + "+" + datetime.datetime.utcnow().strftime("%Y%m%d.%H%M%S"),
+    version={{.project_name}}.__version__ + "+" + local_version,
     url="https://databricks.com",
     author="{{user_name}}",
     description="wheel file based on {{.project_name}}/src",
-    packages=find_packages(where='./src'),
-    package_dir={'': 'src'},
+    packages=find_packages(where="./src"),
+    package_dir={"": "src"},
     entry_points={
         "packages": [
-            "main={{.project_name}}.main:main"
-        ]
+            "main={{.project_name}}.main:main",
+        ],
     },
     install_requires=[
         # Dependencies in case the output wheel file is used as a library dependency.
diff --git a/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl
index 253ed321c..d0286639f 100644
--- a/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl
+++ b/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl
@@ -35,6 +35,7 @@
     "# Import DLT and src/{{.project_name}}\n",
     "import dlt\n",
     "import sys\n",
+    "\n",
     "sys.path.append(spark.conf.get(\"bundle.sourcePath\", \".\"))\n",
     "from pyspark.sql.functions import expr\n",
     "from {{.project_name}} import main"
@@ -63,17 +64,18 @@
     {{- if (eq .include_python "yes") }}
     "@dlt.view\n",
     "def taxi_raw():\n",
-    "  return main.get_taxis(spark)\n",
+    "    return main.get_taxis(spark)\n",
     {{else}}
     "\n",
     "@dlt.view\n",
     "def taxi_raw():\n",
-    "  return spark.read.format(\"json\").load(\"/databricks-datasets/nyctaxi/sample/json/\")\n",
+    "    return spark.read.format(\"json\").load(\"/databricks-datasets/nyctaxi/sample/json/\")\n",
     {{end -}}
     "\n",
+    "\n",
     "@dlt.table\n",
     "def filtered_taxis():\n",
-    "  return dlt.read(\"taxi_raw\").filter(expr(\"fare_amount < 30\"))"
+    "    return dlt.read(\"taxi_raw\").filter(expr(\"fare_amount < 30\"))"
    ]
   }
  ],
diff --git a/libs/template/templates/default-python/template/{{.project_name}}/src/{{.project_name}}/main.py.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/src/{{.project_name}}/main.py.tmpl
index c514c6dc5..5ae344c7e 100644
--- a/libs/template/templates/default-python/template/{{.project_name}}/src/{{.project_name}}/main.py.tmpl
+++ b/libs/template/templates/default-python/template/{{.project_name}}/src/{{.project_name}}/main.py.tmpl
@@ -1,21 +1,25 @@
 from pyspark.sql import SparkSession, DataFrame
 
+
 def get_taxis(spark: SparkSession) -> DataFrame:
-  return spark.read.table("samples.nyctaxi.trips")
+    return spark.read.table("samples.nyctaxi.trips")
 
 
 # Create a new Databricks Connect session. If this fails,
 # check that you have configured Databricks Connect correctly.
 # See https://docs.databricks.com/dev-tools/databricks-connect.html.
 def get_spark() -> SparkSession:
-  try:
-    from databricks.connect import DatabricksSession
-    return DatabricksSession.builder.getOrCreate()
-  except ImportError:
-    return SparkSession.builder.getOrCreate()
+    try:
+        from databricks.connect import DatabricksSession
+
+        return DatabricksSession.builder.getOrCreate()
+    except ImportError:
+        return SparkSession.builder.getOrCreate()
+
 
 def main():
-  get_taxis(get_spark()).show(5)
+    get_taxis(get_spark()).show(5)
 
-if __name__ == '__main__':
-  main()
+
+if __name__ == "__main__":
+    main()

From 5592fa889e8a4c5c2a48ccfbd388180e23b93c58 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 15 Jan 2025 10:45:57 +0100
Subject: [PATCH 079/247] acceptance: add -buildvcs=false when building on
 Windows (#2148)

I get an error on my local Windows otherwise: error obtaining VCS
status: exit status 128

On CI this does not make a difference.
---
 acceptance/acceptance_test.go | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index c7f40d11e..fc486e2cc 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -265,10 +265,23 @@ func BuildCLI(t *testing.T, cwd, coverDir string) string {
 	}
 
 	start := time.Now()
-	args := []string{"go", "build", "-mod", "vendor", "-o", execPath}
+	args := []string{
+		"go", "build",
+		"-mod", "vendor",
+		"-o", execPath,
+	}
+
 	if coverDir != "" {
 		args = append(args, "-cover")
 	}
+
+	if runtime.GOOS == "windows" {
+		// Get this error on my local Windows:
+		// error obtaining VCS status: exit status 128
+		// Use -buildvcs=false to disable VCS stamping.
+		args = append(args, "-buildvcs=false")
+	}
+
 	cmd := exec.Command(args[0], args[1:]...)
 	cmd.Dir = ".."
 	out, err := cmd.CombinedOutput()

From b76eee0e8cf40199cd9fe154b79e101cb05aab9e Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 15 Jan 2025 11:22:23 +0100
Subject: [PATCH 080/247] Migrate resolution tests to acceptance tests (#2143)

---
 .../resolve-nonstrings/databricks.yml         |  23 ++++
 .../variables/resolve-nonstrings/output.txt   |  52 ++++++++
 .../variables/resolve-nonstrings/script       |   4 +
 .../resolve-vars-in-root-path/databricks.yml  |   9 ++
 .../resolve-vars-in-root-path/output.txt      |  11 ++
 .../resolve-vars-in-root-path/script          |   1 +
 .../resolve_variable_references_test.go       | 120 ------------------
 7 files changed, 100 insertions(+), 120 deletions(-)
 create mode 100644 acceptance/bundle/variables/resolve-nonstrings/databricks.yml
 create mode 100644 acceptance/bundle/variables/resolve-nonstrings/output.txt
 create mode 100644 acceptance/bundle/variables/resolve-nonstrings/script
 create mode 100644 acceptance/bundle/variables/resolve-vars-in-root-path/databricks.yml
 create mode 100644 acceptance/bundle/variables/resolve-vars-in-root-path/output.txt
 create mode 100644 acceptance/bundle/variables/resolve-vars-in-root-path/script

diff --git a/acceptance/bundle/variables/resolve-nonstrings/databricks.yml b/acceptance/bundle/variables/resolve-nonstrings/databricks.yml
new file mode 100644
index 000000000..a02c78a7e
--- /dev/null
+++ b/acceptance/bundle/variables/resolve-nonstrings/databricks.yml
@@ -0,0 +1,23 @@
+bundle:
+  name: TestResolveVariableReferencesForPrimitiveNonStringFields
+
+variables:
+  no_alert_for_canceled_runs: {}
+  no_alert_for_skipped_runs: {}
+  min_workers: {}
+  max_workers: {}
+  spot_bid_max_price: {}
+
+resources:
+  jobs:
+    job1:
+      notification_settings:
+        no_alert_for_canceled_runs: ${var.no_alert_for_canceled_runs}
+        no_alert_for_skipped_runs: ${var.no_alert_for_skipped_runs}
+      tasks:
+        - new_cluster:
+            autoscale:
+              min_workers: ${var.min_workers}
+              max_workers: ${var.max_workers}
+            azure_attributes:
+              spot_bid_max_price: ${var.spot_bid_max_price}
diff --git a/acceptance/bundle/variables/resolve-nonstrings/output.txt b/acceptance/bundle/variables/resolve-nonstrings/output.txt
new file mode 100644
index 000000000..3a1eb9c47
--- /dev/null
+++ b/acceptance/bundle/variables/resolve-nonstrings/output.txt
@@ -0,0 +1,52 @@
+{
+  "variables": {
+    "max_workers": {
+      "value": "2"
+    },
+    "min_workers": {
+      "value": "1"
+    },
+    "no_alert_for_canceled_runs": {
+      "value": "true"
+    },
+    "no_alert_for_skipped_runs": {
+      "value": "false"
+    },
+    "spot_bid_max_price": {
+      "value": "0.5"
+    }
+  },
+  "jobs": {
+    "job1": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/TestResolveVariableReferencesForPrimitiveNonStringFields/default/state/metadata.json"
+      },
+      "edit_mode": "UI_LOCKED",
+      "format": "MULTI_TASK",
+      "notification_settings": {
+        "no_alert_for_canceled_runs": true,
+        "no_alert_for_skipped_runs": false
+      },
+      "permissions": [],
+      "queue": {
+        "enabled": true
+      },
+      "tags": {},
+      "tasks": [
+        {
+          "new_cluster": {
+            "autoscale": {
+              "max_workers": 2,
+              "min_workers": 1
+            },
+            "azure_attributes": {
+              "spot_bid_max_price": 0.5
+            }
+          },
+          "task_key": ""
+        }
+      ]
+    }
+  }
+}
diff --git a/acceptance/bundle/variables/resolve-nonstrings/script b/acceptance/bundle/variables/resolve-nonstrings/script
new file mode 100644
index 000000000..cb9e45b61
--- /dev/null
+++ b/acceptance/bundle/variables/resolve-nonstrings/script
@@ -0,0 +1,4 @@
+export BUNDLE_VAR_no_alert_for_skipped_runs=false
+export BUNDLE_VAR_max_workers=2
+export BUNDLE_VAR_min_workers=3  # shadowed by --var below
+$CLI bundle validate -o json --var no_alert_for_canceled_runs=true --var min_workers=1 --var spot_bid_max_price=0.5 | jq '{ variables, jobs: .resources.jobs }'
diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/databricks.yml b/acceptance/bundle/variables/resolve-vars-in-root-path/databricks.yml
new file mode 100644
index 000000000..6a45de330
--- /dev/null
+++ b/acceptance/bundle/variables/resolve-vars-in-root-path/databricks.yml
@@ -0,0 +1,9 @@
+bundle:
+  name: TestResolveVariableReferencesToBundleVariables
+
+workspace:
+  root_path: "${bundle.name}/${var.foo}"
+
+variables:
+  foo:
+    value: "bar"
diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt b/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt
new file mode 100644
index 000000000..c56fbe415
--- /dev/null
+++ b/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt
@@ -0,0 +1,11 @@
+{
+  "artifact_path": "TestResolveVariableReferencesToBundleVariables/bar/artifacts",
+  "current_user": {
+    "short_name": "$USERNAME",
+    "userName": "$USERNAME"
+  },
+  "file_path": "TestResolveVariableReferencesToBundleVariables/bar/files",
+  "resource_path": "TestResolveVariableReferencesToBundleVariables/bar/resources",
+  "root_path": "TestResolveVariableReferencesToBundleVariables/bar",
+  "state_path": "TestResolveVariableReferencesToBundleVariables/bar/state"
+}
diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/script b/acceptance/bundle/variables/resolve-vars-in-root-path/script
new file mode 100644
index 000000000..fefd9abe6
--- /dev/null
+++ b/acceptance/bundle/variables/resolve-vars-in-root-path/script
@@ -0,0 +1 @@
+$CLI bundle validate -o json | jq .workspace
diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go
index 18bb022aa..6bf84ae6d 100644
--- a/bundle/config/mutator/resolve_variable_references_test.go
+++ b/bundle/config/mutator/resolve_variable_references_test.go
@@ -13,129 +13,9 @@ import (
 	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/databricks/databricks-sdk-go/service/pipelines"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func TestResolveVariableReferencesToBundleVariables(t *testing.T) {
-	b := &bundle.Bundle{
-		Config: config.Root{
-			Bundle: config.Bundle{
-				Name: "example",
-			},
-			Workspace: config.Workspace{
-				RootPath: "${bundle.name}/${var.foo}",
-			},
-			Variables: map[string]*variable.Variable{
-				"foo": {
-					Value: "bar",
-				},
-			},
-		},
-	}
-
-	// Apply with a valid prefix. This should change the workspace root path.
-	diags := bundle.Apply(context.Background(), b, ResolveVariableReferences("bundle", "variables"))
-	require.NoError(t, diags.Error())
-	require.Equal(t, "example/bar", b.Config.Workspace.RootPath)
-}
-
-func TestResolveVariableReferencesForPrimitiveNonStringFields(t *testing.T) {
-	var diags diag.Diagnostics
-
-	b := &bundle.Bundle{
-		Config: config.Root{
-			Variables: map[string]*variable.Variable{
-				"no_alert_for_canceled_runs": {},
-				"no_alert_for_skipped_runs":  {},
-				"min_workers":                {},
-				"max_workers":                {},
-				"spot_bid_max_price":         {},
-			},
-			Resources: config.Resources{
-				Jobs: map[string]*resources.Job{
-					"job1": {
-						JobSettings: &jobs.JobSettings{
-							NotificationSettings: &jobs.JobNotificationSettings{
-								NoAlertForCanceledRuns: false,
-								NoAlertForSkippedRuns:  false,
-							},
-							Tasks: []jobs.Task{
-								{
-									NewCluster: &compute.ClusterSpec{
-										Autoscale: &compute.AutoScale{
-											MinWorkers: 0,
-											MaxWorkers: 0,
-										},
-										AzureAttributes: &compute.AzureAttributes{
-											SpotBidMaxPrice: 0.0,
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-
-	ctx := context.Background()
-
-	// Initialize the variables.
-	diags = bundle.ApplyFunc(ctx, b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-		err := b.Config.InitializeVariables([]string{
-			"no_alert_for_canceled_runs=true",
-			"no_alert_for_skipped_runs=true",
-			"min_workers=1",
-			"max_workers=2",
-			"spot_bid_max_price=0.5",
-		})
-		return diag.FromErr(err)
-	})
-	require.NoError(t, diags.Error())
-
-	// Assign the variables to the dynamic configuration.
-	diags = bundle.ApplyFunc(ctx, b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-		err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
-			var p dyn.Path
-			var err error
-
-			// Set the notification settings.
-			p = dyn.MustPathFromString("resources.jobs.job1.notification_settings")
-			v, err = dyn.SetByPath(v, p.Append(dyn.Key("no_alert_for_canceled_runs")), dyn.V("${var.no_alert_for_canceled_runs}"))
-			require.NoError(t, err)
-			v, err = dyn.SetByPath(v, p.Append(dyn.Key("no_alert_for_skipped_runs")), dyn.V("${var.no_alert_for_skipped_runs}"))
-			require.NoError(t, err)
-
-			// Set the min and max workers.
-			p = dyn.MustPathFromString("resources.jobs.job1.tasks[0].new_cluster.autoscale")
-			v, err = dyn.SetByPath(v, p.Append(dyn.Key("min_workers")), dyn.V("${var.min_workers}"))
-			require.NoError(t, err)
-			v, err = dyn.SetByPath(v, p.Append(dyn.Key("max_workers")), dyn.V("${var.max_workers}"))
-			require.NoError(t, err)
-
-			// Set the spot bid max price.
-			p = dyn.MustPathFromString("resources.jobs.job1.tasks[0].new_cluster.azure_attributes")
-			v, err = dyn.SetByPath(v, p.Append(dyn.Key("spot_bid_max_price")), dyn.V("${var.spot_bid_max_price}"))
-			require.NoError(t, err)
-
-			return v, nil
-		})
-		return diag.FromErr(err)
-	})
-	require.NoError(t, diags.Error())
-
-	// Apply for the variable prefix. This should resolve the variables to their values.
-	diags = bundle.Apply(context.Background(), b, ResolveVariableReferences("variables"))
-	require.NoError(t, diags.Error())
-	assert.True(t, b.Config.Resources.Jobs["job1"].JobSettings.NotificationSettings.NoAlertForCanceledRuns)
-	assert.True(t, b.Config.Resources.Jobs["job1"].JobSettings.NotificationSettings.NoAlertForSkippedRuns)
-	assert.Equal(t, 1, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.Autoscale.MinWorkers)
-	assert.Equal(t, 2, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.Autoscale.MaxWorkers)
-	assert.InDelta(t, 0.5, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.AzureAttributes.SpotBidMaxPrice, 0.0001)
-}
-
 func TestResolveComplexVariable(t *testing.T) {
 	b := &bundle.Bundle{
 		Config: config.Root{

From 983a8a66337b4a032343451d6b4b924833eb8bf2 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 15 Jan 2025 11:41:06 +0100
Subject: [PATCH 081/247] Alias 'make' to 'make vendor fmt lint' (#2152)

The current default of building a binary is not frequently used.

The new default is useful after switching branches, stashing/unstashing,
AI changes.

Note "fmt" and "lint" are separate steps because "fmt" can fix
formatting and imports in presence of compilation errors and "lint"
cannot. Without compilation errors, "lint" also does formatting.
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 4e5067b5d..37cf5567c 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-default: build
+default: vendor fmt lint
 
 PACKAGES=./acceptance/... ./libs/... ./internal/... ./cmd/... ./bundle/... .
 

From 40e96b5af2605efad811fb5b1270578632ec6d0a Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 15 Jan 2025 12:05:46 +0100
Subject: [PATCH 082/247] exec(test): Do not clear PATH; this breaks coverage
 on Windows (#2150)

## Changes
When setting up PATH in tests, put desired entry first but keep the rest
as well. Otherwise it fails on Windows

```
D:/a/cli/cli/libs/exec/exec_test.go:108
Error: Received unexpected error:
exit status 0xc0000135
```

Explanation from Claude:
> The error code 0xc0000135 is a Windows error indicating "Unable to
locate DLL"
> When code coverage is enabled, Go instruments the binary with coverage
tracking code, which requires additional DLL dependencies on Windows.

## Tests
Separate draft PR with coverage enabled on CI:
https://github.com/databricks/cli/pull/2141
---
 libs/exec/exec_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libs/exec/exec_test.go b/libs/exec/exec_test.go
index c363c1f7c..f245f9dd1 100644
--- a/libs/exec/exec_test.go
+++ b/libs/exec/exec_test.go
@@ -85,7 +85,7 @@ func testExecutorWithShell(t *testing.T, shell string) {
 
 	// Create temporary directory with only the shell executable in the PATH.
 	tmpDir := t.TempDir()
-	t.Setenv("PATH", tmpDir)
+	t.Setenv("PATH", fmt.Sprintf("%s%c%s", tmpDir, os.PathListSeparator, os.Getenv("PATH")))
 	if runtime.GOOS == "windows" {
 		err = os.Symlink(p, fmt.Sprintf("%s/%s.exe", tmpDir, shell))
 		require.NoError(t, err)

From 626045a17e13ee4b9306fc2f233adfc6f8b25ee1 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Wed, 15 Jan 2025 12:15:23 +0100
Subject: [PATCH 083/247] Use regular expressions for testdiff replacements
 (#2151)

## Changes

Replacement was split between the type `ReplacementContext` and the
`ReplaceOutput` function. The latter also ran a couple of regular
expressions. This change consolidates them such that it is up to the
caller to compose the set of replacements to use.

This change is required to accommodate UUID replacement in #2146.
---
 acceptance/acceptance_test.go                 |   2 +-
 .../bundle/init_default_python_test.go        |   5 +-
 libs/testdiff/context.go                      |  34 ++++
 libs/testdiff/context_test.go                 |  30 ++++
 libs/testdiff/golden.go                       | 150 +----------------
 libs/testdiff/replacement.go                  | 153 ++++++++++++++++++
 libs/testdiff/replacement_test.go             |  46 ++++++
 7 files changed, 269 insertions(+), 151 deletions(-)
 create mode 100644 libs/testdiff/context.go
 create mode 100644 libs/testdiff/context_test.go
 create mode 100644 libs/testdiff/replacement.go
 create mode 100644 libs/testdiff/replacement_test.go

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index fc486e2cc..e6eb380a4 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -88,7 +88,7 @@ func TestAccept(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, user)
 	testdiff.PrepareReplacementsUser(t, &repls, *user)
-	testdiff.PrepareReplacements(t, &repls, workspaceClient)
+	testdiff.PrepareReplacementsWorkspaceClient(t, &repls, workspaceClient)
 
 	testDirs := getTests(t)
 	require.NotEmpty(t, testDirs)
diff --git a/integration/bundle/init_default_python_test.go b/integration/bundle/init_default_python_test.go
index c93e6b50b..931660032 100644
--- a/integration/bundle/init_default_python_test.go
+++ b/integration/bundle/init_default_python_test.go
@@ -58,7 +58,10 @@ func testDefaultPython(t *testing.T, pythonVersion string) {
 	require.NoError(t, err)
 	require.NotNil(t, user)
 	testdiff.PrepareReplacementsUser(t, replacements, *user)
-	testdiff.PrepareReplacements(t, replacements, wt.W)
+	testdiff.PrepareReplacementsWorkspaceClient(t, replacements, wt.W)
+	testdiff.PrepareReplacementsUUID(t, replacements)
+	testdiff.PrepareReplacementsNumber(t, replacements)
+	testdiff.PrepareReplacementsTemporaryDirectory(t, replacements)
 
 	tmpDir := t.TempDir()
 	testutil.Chdir(t, tmpDir)
diff --git a/libs/testdiff/context.go b/libs/testdiff/context.go
new file mode 100644
index 000000000..7b6f5ff88
--- /dev/null
+++ b/libs/testdiff/context.go
@@ -0,0 +1,34 @@
+package testdiff
+
+import (
+	"context"
+)
+
+type key int
+
+const (
+	replacementsMapKey = key(1)
+)
+
+func WithReplacementsMap(ctx context.Context) (context.Context, *ReplacementsContext) {
+	value := ctx.Value(replacementsMapKey)
+	if value != nil {
+		if existingMap, ok := value.(*ReplacementsContext); ok {
+			return ctx, existingMap
+		}
+	}
+
+	newMap := &ReplacementsContext{}
+	ctx = context.WithValue(ctx, replacementsMapKey, newMap)
+	return ctx, newMap
+}
+
+func GetReplacementsMap(ctx context.Context) *ReplacementsContext {
+	value := ctx.Value(replacementsMapKey)
+	if value != nil {
+		if existingMap, ok := value.(*ReplacementsContext); ok {
+			return existingMap
+		}
+	}
+	return nil
+}
diff --git a/libs/testdiff/context_test.go b/libs/testdiff/context_test.go
new file mode 100644
index 000000000..5a0191009
--- /dev/null
+++ b/libs/testdiff/context_test.go
@@ -0,0 +1,30 @@
+package testdiff
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetReplacementsMap_Nil(t *testing.T) {
+	ctx := context.Background()
+	repls := GetReplacementsMap(ctx)
+	assert.Nil(t, repls)
+}
+
+func TestGetReplacementsMap_NotNil(t *testing.T) {
+	ctx := context.Background()
+	ctx, _ = WithReplacementsMap(ctx)
+	repls := GetReplacementsMap(ctx)
+	assert.NotNil(t, repls)
+}
+
+func TestWithReplacementsMap_UseExisting(t *testing.T) {
+	ctx := context.Background()
+	ctx, r1 := WithReplacementsMap(ctx)
+	ctx, r2 := WithReplacementsMap(ctx)
+	repls := GetReplacementsMap(ctx)
+	assert.Equal(t, r1, repls)
+	assert.Equal(t, r2, repls)
+}
diff --git a/libs/testdiff/golden.go b/libs/testdiff/golden.go
index dd07df408..c1c51b6c5 100644
--- a/libs/testdiff/golden.go
+++ b/libs/testdiff/golden.go
@@ -2,25 +2,15 @@ package testdiff
 
 import (
 	"context"
-	"encoding/json"
 	"flag"
-	"fmt"
 	"os"
-	"regexp"
 	"strings"
 	"testing"
 
 	"github.com/databricks/cli/internal/testutil"
-	"github.com/databricks/cli/libs/iamutil"
-	"github.com/databricks/databricks-sdk-go"
-	"github.com/databricks/databricks-sdk-go/service/iam"
 	"github.com/stretchr/testify/assert"
 )
 
-const (
-	testerName = "$USERNAME"
-)
-
 var OverwriteMode = false
 
 func init() {
@@ -75,12 +65,6 @@ func AssertOutputJQ(t testutil.TestingT, ctx context.Context, out, outTitle, exp
 	}
 }
 
-var (
-	uuidRegex        = regexp.MustCompile(`[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}`)
-	numIdRegex       = regexp.MustCompile(`[0-9]{3,}`)
-	privatePathRegex = regexp.MustCompile(`(/tmp|/private)(/.*)/([a-zA-Z0-9]+)`)
-)
-
 func ReplaceOutput(t testutil.TestingT, ctx context.Context, out string) string {
 	t.Helper()
 	out = NormalizeNewlines(out)
@@ -88,139 +72,7 @@ func ReplaceOutput(t testutil.TestingT, ctx context.Context, out string) string
 	if replacements == nil {
 		t.Fatal("WithReplacementsMap was not called")
 	}
-	out = replacements.Replace(out)
-	out = uuidRegex.ReplaceAllString(out, "<UUID>")
-	out = numIdRegex.ReplaceAllString(out, "<NUMID>")
-	out = privatePathRegex.ReplaceAllString(out, "/tmp/.../$3")
-
-	return out
-}
-
-type key int
-
-const (
-	replacementsMapKey = key(1)
-)
-
-type Replacement struct {
-	Old string
-	New string
-}
-
-type ReplacementsContext struct {
-	Repls []Replacement
-}
-
-func (r *ReplacementsContext) Replace(s string) string {
-	// QQQ Should probably only replace whole words
-	for _, repl := range r.Repls {
-		s = strings.ReplaceAll(s, repl.Old, repl.New)
-	}
-	return s
-}
-
-func (r *ReplacementsContext) Set(old, new string) {
-	if old == "" || new == "" {
-		return
-	}
-
-	// Always include both verbatim and json version of replacement.
-	// This helps when the string in question contains \ or other chars that need to be quoted.
-	// In that case we cannot rely that json(old) == '"{old}"' and need to add it explicitly.
-
-	encodedNew, err := json.Marshal(new)
-	if err == nil {
-		encodedOld, err := json.Marshal(old)
-		if err == nil {
-			r.Repls = append(r.Repls, Replacement{Old: string(encodedOld), New: string(encodedNew)})
-		}
-	}
-
-	r.Repls = append(r.Repls, Replacement{Old: old, New: new})
-}
-
-func WithReplacementsMap(ctx context.Context) (context.Context, *ReplacementsContext) {
-	value := ctx.Value(replacementsMapKey)
-	if value != nil {
-		if existingMap, ok := value.(*ReplacementsContext); ok {
-			return ctx, existingMap
-		}
-	}
-
-	newMap := &ReplacementsContext{}
-	ctx = context.WithValue(ctx, replacementsMapKey, newMap)
-	return ctx, newMap
-}
-
-func GetReplacementsMap(ctx context.Context) *ReplacementsContext {
-	value := ctx.Value(replacementsMapKey)
-	if value != nil {
-		if existingMap, ok := value.(*ReplacementsContext); ok {
-			return existingMap
-		}
-	}
-	return nil
-}
-
-func PrepareReplacements(t testutil.TestingT, r *ReplacementsContext, w *databricks.WorkspaceClient) {
-	t.Helper()
-	// in some clouds (gcp) w.Config.Host includes "https://" prefix in others it's really just a host (azure)
-	host := strings.TrimPrefix(strings.TrimPrefix(w.Config.Host, "http://"), "https://")
-	r.Set(host, "$DATABRICKS_HOST")
-	r.Set(w.Config.ClusterID, "$DATABRICKS_CLUSTER_ID")
-	r.Set(w.Config.WarehouseID, "$DATABRICKS_WAREHOUSE_ID")
-	r.Set(w.Config.ServerlessComputeID, "$DATABRICKS_SERVERLESS_COMPUTE_ID")
-	r.Set(w.Config.MetadataServiceURL, "$DATABRICKS_METADATA_SERVICE_URL")
-	r.Set(w.Config.AccountID, "$DATABRICKS_ACCOUNT_ID")
-	r.Set(w.Config.Token, "$DATABRICKS_TOKEN")
-	r.Set(w.Config.Username, "$DATABRICKS_USERNAME")
-	r.Set(w.Config.Password, "$DATABRICKS_PASSWORD")
-	r.Set(w.Config.Profile, "$DATABRICKS_CONFIG_PROFILE")
-	r.Set(w.Config.ConfigFile, "$DATABRICKS_CONFIG_FILE")
-	r.Set(w.Config.GoogleServiceAccount, "$DATABRICKS_GOOGLE_SERVICE_ACCOUNT")
-	r.Set(w.Config.GoogleCredentials, "$GOOGLE_CREDENTIALS")
-	r.Set(w.Config.AzureResourceID, "$DATABRICKS_AZURE_RESOURCE_ID")
-	r.Set(w.Config.AzureClientSecret, "$ARM_CLIENT_SECRET")
-	// r.Set(w.Config.AzureClientID, "$ARM_CLIENT_ID")
-	r.Set(w.Config.AzureClientID, testerName)
-	r.Set(w.Config.AzureTenantID, "$ARM_TENANT_ID")
-	r.Set(w.Config.ActionsIDTokenRequestURL, "$ACTIONS_ID_TOKEN_REQUEST_URL")
-	r.Set(w.Config.ActionsIDTokenRequestToken, "$ACTIONS_ID_TOKEN_REQUEST_TOKEN")
-	r.Set(w.Config.AzureEnvironment, "$ARM_ENVIRONMENT")
-	r.Set(w.Config.ClientID, "$DATABRICKS_CLIENT_ID")
-	r.Set(w.Config.ClientSecret, "$DATABRICKS_CLIENT_SECRET")
-	r.Set(w.Config.DatabricksCliPath, "$DATABRICKS_CLI_PATH")
-	// This is set to words like "path" that happen too frequently
-	// r.Set(w.Config.AuthType, "$DATABRICKS_AUTH_TYPE")
-}
-
-func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam.User) {
-	t.Helper()
-	// There could be exact matches or overlap between different name fields, so sort them by length
-	// to ensure we match the largest one first and map them all to the same token
-
-	r.Set(u.UserName, testerName)
-	r.Set(u.DisplayName, testerName)
-	if u.Name != nil {
-		r.Set(u.Name.FamilyName, testerName)
-		r.Set(u.Name.GivenName, testerName)
-	}
-
-	for _, val := range u.Emails {
-		r.Set(val.Value, testerName)
-	}
-
-	r.Set(iamutil.GetShortUserName(&u), testerName)
-
-	for ind, val := range u.Groups {
-		r.Set(val.Value, fmt.Sprintf("$USER.Groups[%d]", ind))
-	}
-
-	r.Set(u.Id, "$USER.Id")
-
-	for ind, val := range u.Roles {
-		r.Set(val.Value, fmt.Sprintf("$USER.Roles[%d]", ind))
-	}
+	return replacements.Replace(out)
 }
 
 func NormalizeNewlines(input string) string {
diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go
new file mode 100644
index 000000000..fee6d80fd
--- /dev/null
+++ b/libs/testdiff/replacement.go
@@ -0,0 +1,153 @@
+package testdiff
+
+import (
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/databricks/cli/internal/testutil"
+	"github.com/databricks/cli/libs/iamutil"
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/service/iam"
+)
+
+const (
+	testerName = "$USERNAME"
+)
+
+var (
+	uuidRegex        = regexp.MustCompile(`[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}`)
+	numIdRegex       = regexp.MustCompile(`[0-9]{3,}`)
+	privatePathRegex = regexp.MustCompile(`(/tmp|/private)(/.*)/([a-zA-Z0-9]+)`)
+)
+
+type Replacement struct {
+	Old *regexp.Regexp
+	New string
+}
+
+type ReplacementsContext struct {
+	Repls []Replacement
+}
+
+func (r *ReplacementsContext) Replace(s string) string {
+	// QQQ Should probably only replace whole words
+	for _, repl := range r.Repls {
+		s = repl.Old.ReplaceAllString(s, repl.New)
+	}
+	return s
+}
+
+func (r *ReplacementsContext) append(pattern *regexp.Regexp, replacement string) {
+	r.Repls = append(r.Repls, Replacement{
+		Old: pattern,
+		New: replacement,
+	})
+}
+
+func (r *ReplacementsContext) appendLiteral(old, new string) {
+	r.append(
+		// Transform the input strings such that they can be used as literal strings in regular expressions.
+		regexp.MustCompile(regexp.QuoteMeta(old)),
+		// Transform the replacement string such that `$` is interpreted as a literal dollar sign.
+		// For more information about how the replacement string is used, see [regexp.Regexp.Expand].
+		strings.ReplaceAll(new, `$`, `$$`),
+	)
+}
+
+func (r *ReplacementsContext) Set(old, new string) {
+	if old == "" || new == "" {
+		return
+	}
+
+	// Always include both verbatim and json version of replacement.
+	// This helps when the string in question contains \ or other chars that need to be quoted.
+	// In that case we cannot rely that json(old) == '"{old}"' and need to add it explicitly.
+
+	encodedNew, err := json.Marshal(new)
+	if err == nil {
+		encodedOld, err := json.Marshal(old)
+		if err == nil {
+			r.appendLiteral(string(encodedOld), string(encodedNew))
+		}
+	}
+
+	r.appendLiteral(old, new)
+}
+
+func PrepareReplacementsWorkspaceClient(t testutil.TestingT, r *ReplacementsContext, w *databricks.WorkspaceClient) {
+	t.Helper()
+	// in some clouds (gcp) w.Config.Host includes "https://" prefix in others it's really just a host (azure)
+	host := strings.TrimPrefix(strings.TrimPrefix(w.Config.Host, "http://"), "https://")
+	r.Set(host, "$DATABRICKS_HOST")
+	r.Set(w.Config.ClusterID, "$DATABRICKS_CLUSTER_ID")
+	r.Set(w.Config.WarehouseID, "$DATABRICKS_WAREHOUSE_ID")
+	r.Set(w.Config.ServerlessComputeID, "$DATABRICKS_SERVERLESS_COMPUTE_ID")
+	r.Set(w.Config.MetadataServiceURL, "$DATABRICKS_METADATA_SERVICE_URL")
+	r.Set(w.Config.AccountID, "$DATABRICKS_ACCOUNT_ID")
+	r.Set(w.Config.Token, "$DATABRICKS_TOKEN")
+	r.Set(w.Config.Username, "$DATABRICKS_USERNAME")
+	r.Set(w.Config.Password, "$DATABRICKS_PASSWORD")
+	r.Set(w.Config.Profile, "$DATABRICKS_CONFIG_PROFILE")
+	r.Set(w.Config.ConfigFile, "$DATABRICKS_CONFIG_FILE")
+	r.Set(w.Config.GoogleServiceAccount, "$DATABRICKS_GOOGLE_SERVICE_ACCOUNT")
+	r.Set(w.Config.GoogleCredentials, "$GOOGLE_CREDENTIALS")
+	r.Set(w.Config.AzureResourceID, "$DATABRICKS_AZURE_RESOURCE_ID")
+	r.Set(w.Config.AzureClientSecret, "$ARM_CLIENT_SECRET")
+	// r.Set(w.Config.AzureClientID, "$ARM_CLIENT_ID")
+	r.Set(w.Config.AzureClientID, testerName)
+	r.Set(w.Config.AzureTenantID, "$ARM_TENANT_ID")
+	r.Set(w.Config.ActionsIDTokenRequestURL, "$ACTIONS_ID_TOKEN_REQUEST_URL")
+	r.Set(w.Config.ActionsIDTokenRequestToken, "$ACTIONS_ID_TOKEN_REQUEST_TOKEN")
+	r.Set(w.Config.AzureEnvironment, "$ARM_ENVIRONMENT")
+	r.Set(w.Config.ClientID, "$DATABRICKS_CLIENT_ID")
+	r.Set(w.Config.ClientSecret, "$DATABRICKS_CLIENT_SECRET")
+	r.Set(w.Config.DatabricksCliPath, "$DATABRICKS_CLI_PATH")
+	// This is set to words like "path" that happen too frequently
+	// r.Set(w.Config.AuthType, "$DATABRICKS_AUTH_TYPE")
+}
+
+func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam.User) {
+	t.Helper()
+	// There could be exact matches or overlap between different name fields, so sort them by length
+	// to ensure we match the largest one first and map them all to the same token
+
+	r.Set(u.UserName, testerName)
+	r.Set(u.DisplayName, testerName)
+	if u.Name != nil {
+		r.Set(u.Name.FamilyName, testerName)
+		r.Set(u.Name.GivenName, testerName)
+	}
+
+	for _, val := range u.Emails {
+		r.Set(val.Value, testerName)
+	}
+
+	r.Set(iamutil.GetShortUserName(&u), testerName)
+
+	for ind, val := range u.Groups {
+		r.Set(val.Value, fmt.Sprintf("$USER.Groups[%d]", ind))
+	}
+
+	r.Set(u.Id, "$USER.Id")
+
+	for ind, val := range u.Roles {
+		r.Set(val.Value, fmt.Sprintf("$USER.Roles[%d]", ind))
+	}
+}
+
+func PrepareReplacementsUUID(t testutil.TestingT, r *ReplacementsContext) {
+	t.Helper()
+	r.append(uuidRegex, "<UUID>")
+}
+
+func PrepareReplacementsNumber(t testutil.TestingT, r *ReplacementsContext) {
+	t.Helper()
+	r.append(numIdRegex, "<NUMID>")
+}
+
+func PrepareReplacementsTemporaryDirectory(t testutil.TestingT, r *ReplacementsContext) {
+	t.Helper()
+	r.append(privatePathRegex, "/tmp/.../$3")
+}
diff --git a/libs/testdiff/replacement_test.go b/libs/testdiff/replacement_test.go
new file mode 100644
index 000000000..de247c03e
--- /dev/null
+++ b/libs/testdiff/replacement_test.go
@@ -0,0 +1,46 @@
+package testdiff
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReplacement_Literal(t *testing.T) {
+	var repls ReplacementsContext
+
+	repls.Set(`foobar`, `[replacement]`)
+	assert.Equal(t, `[replacement]`, repls.Replace(`foobar`))
+}
+
+func TestReplacement_Encoded(t *testing.T) {
+	var repls ReplacementsContext
+
+	repls.Set(`foo"bar`, `[replacement]`)
+	assert.Equal(t, `"[replacement]"`, repls.Replace(`"foo\"bar"`))
+}
+
+func TestReplacement_UUID(t *testing.T) {
+	var repls ReplacementsContext
+
+	PrepareReplacementsUUID(t, &repls)
+
+	assert.Equal(t, "<UUID>", repls.Replace("123e4567-e89b-12d3-a456-426614174000"))
+}
+
+func TestReplacement_Number(t *testing.T) {
+	var repls ReplacementsContext
+
+	PrepareReplacementsNumber(t, &repls)
+
+	assert.Equal(t, "12", repls.Replace("12"))
+	assert.Equal(t, "<NUMID>", repls.Replace("123"))
+}
+
+func TestReplacement_TemporaryDirectory(t *testing.T) {
+	var repls ReplacementsContext
+
+	PrepareReplacementsTemporaryDirectory(t, &repls)
+
+	assert.Equal(t, "/tmp/.../tail", repls.Replace("/tmp/foo/bar/qux/tail"))
+}

From dd554412a67684e5c34622c5cffb95a6a60c1127 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Wed, 15 Jan 2025 12:51:06 +0100
Subject: [PATCH 084/247] Retry app deployment if there is an active deployment
 in progress (#2153)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Changes
If before running an app, the app was stopped with an active deployment,
then Apps backend start it and does the auto-deploy of the last active
deployment. In such cases StartApp API won't return any active or
pending deployments in its response but doing the deploy immediately
after the start might result in the error `Cannot deploy app *** as
there is an active deployment in progress`.

From DABs side, we have to do a new deployment on every `bundle run`
(command which start the app and does deployment) because local files in
bundle might have been changed and users expect to have the app running
with new code.

Thus this PR works around the error by catching “deployment in progress”
error, getting any active / pending deployments, waits for them to
finish and start the new deployment again. If 2nd attempts fails, the
whole command fails.

## Tests
Added unit test.
---
 bundle/run/app.go      | 52 +++++++++++++++++++++++++++-------
 bundle/run/app_test.go | 64 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 106 insertions(+), 10 deletions(-)

diff --git a/bundle/run/app.go b/bundle/run/app.go
index 11030beda..b15f3f4b6 100644
--- a/bundle/run/app.go
+++ b/bundle/run/app.go
@@ -10,6 +10,7 @@ import (
 	"github.com/databricks/cli/bundle/config/resources"
 	"github.com/databricks/cli/bundle/run/output"
 	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/service/apps"
 	"github.com/spf13/cobra"
 )
@@ -111,11 +112,21 @@ func (a *appRunner) start(ctx context.Context) error {
 	// active and pending deployments fields (if any). If there are active or pending deployments,
 	// we need to wait for them to complete before we can do the new deployment.
 	// Otherwise, the new deployment will fail.
-	// Thus, we first wait for the active deployment to complete.
-	if startedApp.ActiveDeployment != nil &&
-		startedApp.ActiveDeployment.Status.State == apps.AppDeploymentStateInProgress {
+	err = waitForDeploymentToComplete(ctx, w, startedApp)
+	if err != nil {
+		return err
+	}
+
+	logProgress(ctx, "App is started!")
+	return nil
+}
+
+func waitForDeploymentToComplete(ctx context.Context, w *databricks.WorkspaceClient, app *apps.App) error {
+	// We first wait for the active deployment to complete.
+	if app.ActiveDeployment != nil &&
+		app.ActiveDeployment.Status.State == apps.AppDeploymentStateInProgress {
 		logProgress(ctx, "Waiting for the active deployment to complete...")
-		_, err = w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, startedApp.ActiveDeployment.DeploymentId, 20*time.Minute, nil)
+		_, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.ActiveDeployment.DeploymentId, 20*time.Minute, nil)
 		if err != nil {
 			return err
 		}
@@ -123,17 +134,16 @@ func (a *appRunner) start(ctx context.Context) error {
 	}
 
 	// Then, we wait for the pending deployment to complete.
-	if startedApp.PendingDeployment != nil &&
-		startedApp.PendingDeployment.Status.State == apps.AppDeploymentStateInProgress {
+	if app.PendingDeployment != nil &&
+		app.PendingDeployment.Status.State == apps.AppDeploymentStateInProgress {
 		logProgress(ctx, "Waiting for the pending deployment to complete...")
-		_, err = w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, startedApp.PendingDeployment.DeploymentId, 20*time.Minute, nil)
+		_, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.PendingDeployment.DeploymentId, 20*time.Minute, nil)
 		if err != nil {
 			return err
 		}
 		logProgress(ctx, "Pending deployment is completed!")
 	}
 
-	logProgress(ctx, "App is started!")
 	return nil
 }
 
@@ -142,16 +152,38 @@ func (a *appRunner) deploy(ctx context.Context) error {
 	b := a.bundle
 	w := b.WorkspaceClient()
 
+	sourceCodePath := app.SourceCodePath
 	wait, err := w.Apps.Deploy(ctx, apps.CreateAppDeploymentRequest{
 		AppName: app.Name,
 		AppDeployment: &apps.AppDeployment{
 			Mode:           apps.AppDeploymentModeSnapshot,
-			SourceCodePath: app.SourceCodePath,
+			SourceCodePath: sourceCodePath,
 		},
 	})
 	// If deploy returns an error, then there's an active deployment in progress, wait for it to complete.
+	// For this we first need to get an app and its acrive and pending deployments and then wait for them.
 	if err != nil {
-		return err
+		app, err := w.Apps.Get(ctx, apps.GetAppRequest{Name: app.Name})
+		if err != nil {
+			return fmt.Errorf("failed to get app %s: %w", app.Name, err)
+		}
+
+		err = waitForDeploymentToComplete(ctx, w, app)
+		if err != nil {
+			return err
+		}
+
+		// Now we can try to deploy the app again
+		wait, err = w.Apps.Deploy(ctx, apps.CreateAppDeploymentRequest{
+			AppName: app.Name,
+			AppDeployment: &apps.AppDeployment{
+				Mode:           apps.AppDeploymentModeSnapshot,
+				SourceCodePath: sourceCodePath,
+			},
+		})
+		if err != nil {
+			return err
+		}
 	}
 
 	_, err = wait.OnProgress(func(ad *apps.AppDeployment) {
diff --git a/bundle/run/app_test.go b/bundle/run/app_test.go
index 44ff698e5..8e82f45ae 100644
--- a/bundle/run/app_test.go
+++ b/bundle/run/app_test.go
@@ -3,6 +3,7 @@ package run
 import (
 	"bytes"
 	"context"
+	"errors"
 	"os"
 	"path/filepath"
 	"testing"
@@ -189,6 +190,69 @@ func TestAppRunWithAnActiveDeploymentInProgress(t *testing.T) {
 	r.run(t)
 }
 
+func TestAppDeployWithDeploymentInProgress(t *testing.T) {
+	ctx, b, mwc := setupBundle(t)
+
+	appApi := mwc.GetMockAppsAPI()
+	appApi.EXPECT().Get(mock.Anything, apps.GetAppRequest{
+		Name: "my_app",
+	}).Return(&apps.App{
+		Name: "my_app",
+		AppStatus: &apps.ApplicationStatus{
+			State: apps.ApplicationStateRunning,
+		},
+		ComputeStatus: &apps.ComputeStatus{
+			State: apps.ComputeStateActive,
+		},
+	}, nil).Once()
+
+	wait := &apps.WaitGetDeploymentAppSucceeded[apps.AppDeployment]{
+		Poll: func(_ time.Duration, _ func(*apps.AppDeployment)) (*apps.AppDeployment, error) {
+			return nil, nil
+		},
+	}
+
+	// First deployment fails
+	appApi.EXPECT().Deploy(mock.Anything, apps.CreateAppDeploymentRequest{
+		AppName: "my_app",
+		AppDeployment: &apps.AppDeployment{
+			Mode:           apps.AppDeploymentModeSnapshot,
+			SourceCodePath: "/Workspace/Users/foo@bar.com/files/my_app",
+		},
+	}).Return(nil, errors.New("deployment in progress")).Once()
+
+	// After first deployment fails, we should get the app and wait for the deployment to complete
+	appApi.EXPECT().Get(mock.Anything, apps.GetAppRequest{
+		Name: "my_app",
+	}).Return(&apps.App{
+		Name: "my_app",
+		ActiveDeployment: &apps.AppDeployment{
+			DeploymentId: "active_deployment_id",
+			Status: &apps.AppDeploymentStatus{
+				State: apps.AppDeploymentStateInProgress,
+			},
+		},
+	}, nil).Once()
+
+	appApi.EXPECT().WaitGetDeploymentAppSucceeded(mock.Anything, "my_app", "active_deployment_id", mock.Anything, mock.Anything).Return(nil, nil)
+
+	// Second one should succeeed
+	appApi.EXPECT().Deploy(mock.Anything, apps.CreateAppDeploymentRequest{
+		AppName: "my_app",
+		AppDeployment: &apps.AppDeployment{
+			Mode:           apps.AppDeploymentModeSnapshot,
+			SourceCodePath: "/Workspace/Users/foo@bar.com/files/my_app",
+		},
+	}).Return(wait, nil).Once()
+
+	r := &testAppRunner{
+		m:   mwc,
+		b:   b,
+		ctx: ctx,
+	}
+	r.run(t)
+}
+
 func TestStopApp(t *testing.T) {
 	ctx, b, mwc := setupBundle(t)
 	appsApi := mwc.GetMockAppsAPI()

From 581565a1c4331686d706914b9b42aa288c45f81c Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 15 Jan 2025 15:59:42 +0100
Subject: [PATCH 085/247] Migrate more variable tests to acceptance (#2154)

---
 .../variables/complex-simple/databricks.yml   |  27 ++++
 .../variables/complex-simple/output.txt       |  10 ++
 .../bundle/variables/complex-simple/script    |   1 +
 .../complex-within-complex/databricks.yml     |  34 +++++
 .../complex-within-complex/output.txt         |  18 +++
 .../variables/complex-within-complex/script   |   1 +
 .../resolve_variable_references_test.go       | 122 ------------------
 7 files changed, 91 insertions(+), 122 deletions(-)
 create mode 100644 acceptance/bundle/variables/complex-simple/databricks.yml
 create mode 100644 acceptance/bundle/variables/complex-simple/output.txt
 create mode 100644 acceptance/bundle/variables/complex-simple/script
 create mode 100644 acceptance/bundle/variables/complex-within-complex/databricks.yml
 create mode 100644 acceptance/bundle/variables/complex-within-complex/output.txt
 create mode 100644 acceptance/bundle/variables/complex-within-complex/script

diff --git a/acceptance/bundle/variables/complex-simple/databricks.yml b/acceptance/bundle/variables/complex-simple/databricks.yml
new file mode 100644
index 000000000..135ff86cf
--- /dev/null
+++ b/acceptance/bundle/variables/complex-simple/databricks.yml
@@ -0,0 +1,27 @@
+# This example works and properly merges resources.jobs.job1.job_clusters.new_cluster and ${var.cluster}.
+# retaining num_workers, spark_version and overriding node_type_id.
+bundle:
+  name: TestResolveComplexVariable
+
+variables:
+  cluster:
+    type: "complex"
+    value:
+      node_type_id: "Standard_DS3_v2"
+      num_workers: 2
+
+resources:
+  jobs:
+    job1:
+      job_clusters:
+        - new_cluster:
+            node_type_id: "random"
+            spark_version: 13.3.x-scala2.12
+
+targets:
+  dev:
+    resources:
+      jobs:
+        job1:
+          job_clusters:
+            - new_cluster: ${var.cluster}
diff --git a/acceptance/bundle/variables/complex-simple/output.txt b/acceptance/bundle/variables/complex-simple/output.txt
new file mode 100644
index 000000000..16b0ec80f
--- /dev/null
+++ b/acceptance/bundle/variables/complex-simple/output.txt
@@ -0,0 +1,10 @@
+[
+  {
+    "job_cluster_key": "",
+    "new_cluster": {
+      "node_type_id": "Standard_DS3_v2",
+      "num_workers": 2,
+      "spark_version": "13.3.x-scala2.12"
+    }
+  }
+]
diff --git a/acceptance/bundle/variables/complex-simple/script b/acceptance/bundle/variables/complex-simple/script
new file mode 100644
index 000000000..1c31d0b40
--- /dev/null
+++ b/acceptance/bundle/variables/complex-simple/script
@@ -0,0 +1 @@
+$CLI bundle validate -o json | jq .resources.jobs.job1.job_clusters
diff --git a/acceptance/bundle/variables/complex-within-complex/databricks.yml b/acceptance/bundle/variables/complex-within-complex/databricks.yml
new file mode 100644
index 000000000..f1d77289e
--- /dev/null
+++ b/acceptance/bundle/variables/complex-within-complex/databricks.yml
@@ -0,0 +1,34 @@
+# Does not work currently, explicitly disabled, even though it works if you remove 'type: "complex"' lines
+# Also fails to merge clusters.
+bundle:
+  name: TestResolveComplexVariableReferencesWithComplexVariablesError
+
+variables:
+  cluster:
+    type: "complex"
+    value:
+      node_type_id: "Standard_DS3_v2"
+      num_workers: 2
+      spark_conf: "${var.spark_conf}"
+  spark_conf:
+    type: "complex"
+    value:
+      spark.executor.memory: "4g"
+      spark.executor.cores: "2"
+
+resources:
+  jobs:
+    job1:
+      job_clusters:
+        - job_cluster_key: my_cluster
+          new_cluster:
+          node_type_id: "random"
+
+targets:
+  dev:
+    resources:
+      jobs:
+        job1:
+          job_clusters:
+            - job_cluster_key: my_cluster
+              new_cluster: ${var.cluster}
diff --git a/acceptance/bundle/variables/complex-within-complex/output.txt b/acceptance/bundle/variables/complex-within-complex/output.txt
new file mode 100644
index 000000000..124bc3363
--- /dev/null
+++ b/acceptance/bundle/variables/complex-within-complex/output.txt
@@ -0,0 +1,18 @@
+Warning: unknown field: node_type_id
+  at resources.jobs.job1.job_clusters[0]
+  in databricks.yml:25:11
+
+Error: complex variables cannot contain references to another complex variables
+
+[
+  {
+    "job_cluster_key": "my_cluster",
+    "new_cluster": null
+  },
+  {
+    "job_cluster_key": "my_cluster",
+    "new_cluster": "${var.cluster}"
+  }
+]
+
+Exit code: 1
diff --git a/acceptance/bundle/variables/complex-within-complex/script b/acceptance/bundle/variables/complex-within-complex/script
new file mode 100644
index 000000000..1c31d0b40
--- /dev/null
+++ b/acceptance/bundle/variables/complex-within-complex/script
@@ -0,0 +1 @@
+$CLI bundle validate -o json | jq .resources.jobs.job1.job_clusters
diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go
index 6bf84ae6d..4bae7867a 100644
--- a/bundle/config/mutator/resolve_variable_references_test.go
+++ b/bundle/config/mutator/resolve_variable_references_test.go
@@ -16,128 +16,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestResolveComplexVariable(t *testing.T) {
-	b := &bundle.Bundle{
-		Config: config.Root{
-			Bundle: config.Bundle{
-				Name: "example",
-			},
-			Variables: map[string]*variable.Variable{
-				"cluster": {
-					Value: map[string]any{
-						"node_type_id": "Standard_DS3_v2",
-						"num_workers":  2,
-					},
-					Type: variable.VariableTypeComplex,
-				},
-			},
-
-			Resources: config.Resources{
-				Jobs: map[string]*resources.Job{
-					"job1": {
-						JobSettings: &jobs.JobSettings{
-							JobClusters: []jobs.JobCluster{
-								{
-									NewCluster: compute.ClusterSpec{
-										NodeTypeId: "random",
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-
-	ctx := context.Background()
-
-	// Assign the variables to the dynamic configuration.
-	diags := bundle.ApplyFunc(ctx, b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-		err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
-			var p dyn.Path
-			var err error
-
-			p = dyn.MustPathFromString("resources.jobs.job1.job_clusters[0]")
-			v, err = dyn.SetByPath(v, p.Append(dyn.Key("new_cluster")), dyn.V("${var.cluster}"))
-			require.NoError(t, err)
-
-			return v, nil
-		})
-		return diag.FromErr(err)
-	})
-	require.NoError(t, diags.Error())
-
-	diags = bundle.Apply(ctx, b, ResolveVariableReferences("bundle", "workspace", "variables"))
-	require.NoError(t, diags.Error())
-	require.Equal(t, "Standard_DS3_v2", b.Config.Resources.Jobs["job1"].JobSettings.JobClusters[0].NewCluster.NodeTypeId)
-	require.Equal(t, 2, b.Config.Resources.Jobs["job1"].JobSettings.JobClusters[0].NewCluster.NumWorkers)
-}
-
-func TestResolveComplexVariableReferencesWithComplexVariablesError(t *testing.T) {
-	b := &bundle.Bundle{
-		Config: config.Root{
-			Bundle: config.Bundle{
-				Name: "example",
-			},
-			Variables: map[string]*variable.Variable{
-				"cluster": {
-					Value: map[string]any{
-						"node_type_id": "Standard_DS3_v2",
-						"num_workers":  2,
-						"spark_conf":   "${var.spark_conf}",
-					},
-					Type: variable.VariableTypeComplex,
-				},
-				"spark_conf": {
-					Value: map[string]any{
-						"spark.executor.memory": "4g",
-						"spark.executor.cores":  "2",
-					},
-					Type: variable.VariableTypeComplex,
-				},
-			},
-
-			Resources: config.Resources{
-				Jobs: map[string]*resources.Job{
-					"job1": {
-						JobSettings: &jobs.JobSettings{
-							JobClusters: []jobs.JobCluster{
-								{
-									NewCluster: compute.ClusterSpec{
-										NodeTypeId: "random",
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-
-	ctx := context.Background()
-
-	// Assign the variables to the dynamic configuration.
-	diags := bundle.ApplyFunc(ctx, b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-		err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
-			var p dyn.Path
-			var err error
-
-			p = dyn.MustPathFromString("resources.jobs.job1.job_clusters[0]")
-			v, err = dyn.SetByPath(v, p.Append(dyn.Key("new_cluster")), dyn.V("${var.cluster}"))
-			require.NoError(t, err)
-
-			return v, nil
-		})
-		return diag.FromErr(err)
-	})
-	require.NoError(t, diags.Error())
-
-	diags = bundle.Apply(ctx, b, bundle.Seq(ResolveVariableReferencesInComplexVariables(), ResolveVariableReferences("bundle", "workspace", "variables")))
-	require.ErrorContains(t, diags.Error(), "complex variables cannot contain references to another complex variables")
-}
-
 func TestResolveComplexVariableWithVarReference(t *testing.T) {
 	b := &bundle.Bundle{
 		Config: config.Root{

From 20179457b99bceac79c41d8acb3b553a1042ca50 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Wed, 15 Jan 2025 17:02:58 +0100
Subject: [PATCH 086/247] Process all the fields in top level request object
 even if it contains request body (#2155)

## Changes

When regenerating CLI with a new Go SDK
https://github.com/databricks/cli/pull/2126 I've noticed that some
parameters such as `no_compute` for apps are not added as flags for the
CLI commands.

This happened because we ignored all other top level fields if there's a
request body object field.

This PR relies on new AllFields method from Genkit which returns fields
from both request object and request body object.
---
 .codegen/service.go.tmpl                                      | 4 ++--
 cmd/account/federation-policy/federation-policy.go            | 2 ++
 .../service-principal-federation-policy.go                    | 2 ++
 cmd/workspace/apps/apps.go                                    | 1 +
 4 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/.codegen/service.go.tmpl b/.codegen/service.go.tmpl
index ee2c7b0fd..0c9fa089a 100644
--- a/.codegen/service.go.tmpl
+++ b/.codegen/service.go.tmpl
@@ -140,9 +140,9 @@ func new{{.PascalName}}() *cobra.Command {
 	{{- end}}
     {{$method := .}}
 	{{ if not .IsJsonOnly }}
-	{{range $request.Fields -}}
+	{{range .AllFields -}}
 		{{- if not .Required -}}
-			{{if .Entity.IsObject }}// TODO: complex arg: {{.Name}}
+			{{if .Entity.IsObject}}{{if not (eq . $method.RequestBodyField) }}// TODO: complex arg: {{.Name}}{{end}}
 			{{else if .Entity.IsAny }}// TODO: any: {{.Name}}
 			{{else if .Entity.ArrayValue }}// TODO: array: {{.Name}}
 			{{else if .Entity.MapValue }}// TODO: map via StringToStringVar: {{.Name}}
diff --git a/cmd/account/federation-policy/federation-policy.go b/cmd/account/federation-policy/federation-policy.go
index 897532e3a..e47bf8324 100755
--- a/cmd/account/federation-policy/federation-policy.go
+++ b/cmd/account/federation-policy/federation-policy.go
@@ -110,6 +110,7 @@ func newCreate() *cobra.Command {
 	// TODO: short flags
 	cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
+	cmd.Flags().StringVar(&createReq.PolicyId, "policy-id", createReq.PolicyId, `The identifier for the federation policy.`)
 	cmd.Flags().StringVar(&createReq.Policy.Description, "description", createReq.Policy.Description, `Description of the federation policy.`)
 	cmd.Flags().StringVar(&createReq.Policy.Name, "name", createReq.Policy.Name, `Resource name for the federation policy.`)
 	// TODO: complex arg: oidc_policy
@@ -344,6 +345,7 @@ func newUpdate() *cobra.Command {
 	// TODO: short flags
 	cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
+	cmd.Flags().StringVar(&updateReq.UpdateMask, "update-mask", updateReq.UpdateMask, `The field mask specifies which fields of the policy to update.`)
 	cmd.Flags().StringVar(&updateReq.Policy.Description, "description", updateReq.Policy.Description, `Description of the federation policy.`)
 	cmd.Flags().StringVar(&updateReq.Policy.Name, "name", updateReq.Policy.Name, `Resource name for the federation policy.`)
 	// TODO: complex arg: oidc_policy
diff --git a/cmd/account/service-principal-federation-policy/service-principal-federation-policy.go b/cmd/account/service-principal-federation-policy/service-principal-federation-policy.go
index 5f33505a1..df36de239 100755
--- a/cmd/account/service-principal-federation-policy/service-principal-federation-policy.go
+++ b/cmd/account/service-principal-federation-policy/service-principal-federation-policy.go
@@ -117,6 +117,7 @@ func newCreate() *cobra.Command {
 	// TODO: short flags
 	cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
+	cmd.Flags().StringVar(&createReq.PolicyId, "policy-id", createReq.PolicyId, `The identifier for the federation policy.`)
 	cmd.Flags().StringVar(&createReq.Policy.Description, "description", createReq.Policy.Description, `Description of the federation policy.`)
 	cmd.Flags().StringVar(&createReq.Policy.Name, "name", createReq.Policy.Name, `Resource name for the federation policy.`)
 	// TODO: complex arg: oidc_policy
@@ -376,6 +377,7 @@ func newUpdate() *cobra.Command {
 	// TODO: short flags
 	cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
+	cmd.Flags().StringVar(&updateReq.UpdateMask, "update-mask", updateReq.UpdateMask, `The field mask specifies which fields of the policy to update.`)
 	cmd.Flags().StringVar(&updateReq.Policy.Description, "description", updateReq.Policy.Description, `Description of the federation policy.`)
 	cmd.Flags().StringVar(&updateReq.Policy.Name, "name", updateReq.Policy.Name, `Resource name for the federation policy.`)
 	// TODO: complex arg: oidc_policy
diff --git a/cmd/workspace/apps/apps.go b/cmd/workspace/apps/apps.go
index a103ba7a8..f7c08ece1 100755
--- a/cmd/workspace/apps/apps.go
+++ b/cmd/workspace/apps/apps.go
@@ -78,6 +78,7 @@ func newCreate() *cobra.Command {
 	// TODO: short flags
 	cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
+	cmd.Flags().BoolVar(&createReq.NoCompute, "no-compute", createReq.NoCompute, `If true, the app will not be started after creation.`)
 	// TODO: complex arg: active_deployment
 	// TODO: complex arg: app_status
 	// TODO: complex arg: compute_status

From d53a78e92651e0e454b4abd608d5e338a476e2c4 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 15 Jan 2025 17:24:12 +0100
Subject: [PATCH 087/247] Introduce $DATABRICKS_URL replacement in tests
 (#2158)

## Changes
It covers both https://$DATABRICKS_HOST and http://$DATABRICKS_HOST so
the test output does not change between local and the cloud.

## Tests
Existing tests using golden files (acceptance and integration) catch
this and were updated.
---
 acceptance/bundle/init/dbt-sql/output.txt                   | 6 +++---
 acceptance/bundle/init/default-python/output.txt            | 6 +++---
 acceptance/bundle/init/default-sql/output.txt               | 6 +++---
 integration/bundle/testdata/default_python/bundle_init.txt  | 2 +-
 .../bundle/testdata/default_python/bundle_summary.txt       | 6 +++---
 .../bundle/testdata/default_python/bundle_validate.txt      | 2 +-
 libs/testdiff/replacement.go                                | 2 ++
 7 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/acceptance/bundle/init/dbt-sql/output.txt b/acceptance/bundle/init/dbt-sql/output.txt
index 384013d14..4f6f83693 100644
--- a/acceptance/bundle/init/dbt-sql/output.txt
+++ b/acceptance/bundle/init/dbt-sql/output.txt
@@ -4,7 +4,7 @@
 Welcome to the dbt template for Databricks Asset Bundles!
 
 A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html.
-workspace_host: http://$DATABRICKS_HOST
+workspace_host: $DATABRICKS_URL
 
 📊 Your new project has been created in the 'my_dbt_sql' directory!
 If you already have dbt installed, just type 'cd my_dbt_sql; dbt init' to get started.
@@ -15,7 +15,7 @@ Refer to the README.md file for full "getting started" guide and production setu
 Name: my_dbt_sql
 Target: dev
 Workspace:
-  Host: http://$DATABRICKS_HOST
+  Host: $DATABRICKS_URL
   User: $USERNAME
   Path: /Workspace/Users/$USERNAME/.bundle/my_dbt_sql/dev
 
@@ -25,7 +25,7 @@ Validation OK!
 Name: my_dbt_sql
 Target: prod
 Workspace:
-  Host: http://$DATABRICKS_HOST
+  Host: $DATABRICKS_URL
   User: $USERNAME
   Path: /Workspace/Users/$USERNAME/.bundle/my_dbt_sql/prod
 
diff --git a/acceptance/bundle/init/default-python/output.txt b/acceptance/bundle/init/default-python/output.txt
index c389ef0ad..a02a7d41b 100644
--- a/acceptance/bundle/init/default-python/output.txt
+++ b/acceptance/bundle/init/default-python/output.txt
@@ -2,7 +2,7 @@
 >>> $CLI bundle init default-python --config-file ./input.json
 
 Welcome to the default Python template for Databricks Asset Bundles!
-Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): http://$DATABRICKS_HOST
+Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): $DATABRICKS_URL
 
 ✨ Your new project has been created in the 'my_default_python' directory!
 
@@ -13,7 +13,7 @@ See also the documentation at https://docs.databricks.com/dev-tools/bundles/inde
 Name: my_default_python
 Target: dev
 Workspace:
-  Host: http://$DATABRICKS_HOST
+  Host: $DATABRICKS_URL
   User: $USERNAME
   Path: /Workspace/Users/$USERNAME/.bundle/my_default_python/dev
 
@@ -23,7 +23,7 @@ Validation OK!
 Name: my_default_python
 Target: prod
 Workspace:
-  Host: http://$DATABRICKS_HOST
+  Host: $DATABRICKS_URL
   User: $USERNAME
   Path: /Workspace/Users/$USERNAME/.bundle/my_default_python/prod
 
diff --git a/acceptance/bundle/init/default-sql/output.txt b/acceptance/bundle/init/default-sql/output.txt
index 3605206dc..ff107eae0 100644
--- a/acceptance/bundle/init/default-sql/output.txt
+++ b/acceptance/bundle/init/default-sql/output.txt
@@ -4,7 +4,7 @@
 Welcome to the default SQL template for Databricks Asset Bundles!
 
 A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html.
-workspace_host: http://$DATABRICKS_HOST
+workspace_host: $DATABRICKS_URL
 
 ✨ Your new project has been created in the 'my_default_sql' directory!
 
@@ -15,7 +15,7 @@ See also the documentation at https://docs.databricks.com/dev-tools/bundles/inde
 Name: my_default_sql
 Target: dev
 Workspace:
-  Host: http://$DATABRICKS_HOST
+  Host: $DATABRICKS_URL
   User: $USERNAME
   Path: /Workspace/Users/$USERNAME/.bundle/my_default_sql/dev
 
@@ -25,7 +25,7 @@ Validation OK!
 Name: my_default_sql
 Target: prod
 Workspace:
-  Host: http://$DATABRICKS_HOST
+  Host: $DATABRICKS_URL
   User: $USERNAME
   Path: /Workspace/Users/$USERNAME/.bundle/my_default_sql/prod
 
diff --git a/integration/bundle/testdata/default_python/bundle_init.txt b/integration/bundle/testdata/default_python/bundle_init.txt
index 6cfc32f98..c2917ea4e 100644
--- a/integration/bundle/testdata/default_python/bundle_init.txt
+++ b/integration/bundle/testdata/default_python/bundle_init.txt
@@ -1,6 +1,6 @@
 
 Welcome to the default Python template for Databricks Asset Bundles!
-Workspace to use (auto-detected, edit in 'project_name_$UNIQUE_PRJ/databricks.yml'): https://$DATABRICKS_HOST
+Workspace to use (auto-detected, edit in 'project_name_$UNIQUE_PRJ/databricks.yml'): $DATABRICKS_URL
 
 ✨ Your new project has been created in the 'project_name_$UNIQUE_PRJ' directory!
 
diff --git a/integration/bundle/testdata/default_python/bundle_summary.txt b/integration/bundle/testdata/default_python/bundle_summary.txt
index a0bcfdbc8..318cd2543 100644
--- a/integration/bundle/testdata/default_python/bundle_summary.txt
+++ b/integration/bundle/testdata/default_python/bundle_summary.txt
@@ -23,7 +23,7 @@
     "resources/project_name_$UNIQUE_PRJ.pipeline.yml"
   ],
   "workspace": {
-    "host": "https://$DATABRICKS_HOST",
+    "host": "$DATABRICKS_URL",
     "current_user": {
       "active": true,
       "displayName": "$USERNAME",
@@ -141,7 +141,7 @@
             "unit": "DAYS"
           }
         },
-        "url": "https://$DATABRICKS_HOST/jobs/<NUMID>?o=<NUMID>"
+        "url": "$DATABRICKS_URL/jobs/<NUMID>?o=<NUMID>"
       }
     },
     "pipelines": {
@@ -165,7 +165,7 @@
         ],
         "name": "[dev $USERNAME] project_name_$UNIQUE_PRJ_pipeline",
         "target": "project_name_$UNIQUE_PRJ_dev",
-        "url": "https://$DATABRICKS_HOST/pipelines/<UUID>?o=<NUMID>"
+        "url": "$DATABRICKS_URL/pipelines/<UUID>?o=<NUMID>"
       }
     }
   },
diff --git a/integration/bundle/testdata/default_python/bundle_validate.txt b/integration/bundle/testdata/default_python/bundle_validate.txt
index 88a5fdd18..578fd6494 100644
--- a/integration/bundle/testdata/default_python/bundle_validate.txt
+++ b/integration/bundle/testdata/default_python/bundle_validate.txt
@@ -1,7 +1,7 @@
 Name: project_name_$UNIQUE_PRJ
 Target: dev
 Workspace:
-  Host: https://$DATABRICKS_HOST
+  Host: $DATABRICKS_URL
   User: $USERNAME
   Path: /Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev
 
diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go
index fee6d80fd..1ab976109 100644
--- a/libs/testdiff/replacement.go
+++ b/libs/testdiff/replacement.go
@@ -80,6 +80,8 @@ func PrepareReplacementsWorkspaceClient(t testutil.TestingT, r *ReplacementsCont
 	t.Helper()
 	// in some clouds (gcp) w.Config.Host includes "https://" prefix in others it's really just a host (azure)
 	host := strings.TrimPrefix(strings.TrimPrefix(w.Config.Host, "http://"), "https://")
+	r.Set("https://"+host, "$DATABRICKS_URL")
+	r.Set("http://"+host, "$DATABRICKS_URL")
 	r.Set(host, "$DATABRICKS_HOST")
 	r.Set(w.Config.ClusterID, "$DATABRICKS_CLUSTER_ID")
 	r.Set(w.Config.WarehouseID, "$DATABRICKS_WAREHOUSE_ID")

From 39b03592d7c79a93825a97312b2f13b06b0e683c Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 15 Jan 2025 17:52:17 +0100
Subject: [PATCH 088/247] Migrate TestResolveComplexVariableWithVarReference
 (#2156)

This is the last test referencing
ResolveVariableReferencesInComplexVariables, allowing removal of that
mutator.
---
 .../complex-with-var-reference/databricks.yml | 17 +++++
 .../complex-with-var-reference/output.txt     | 12 ++++
 .../complex-with-var-reference/script         |  1 +
 .../resolve_variable_references_test.go       | 69 -------------------
 4 files changed, 30 insertions(+), 69 deletions(-)
 create mode 100644 acceptance/bundle/variables/complex-with-var-reference/databricks.yml
 create mode 100644 acceptance/bundle/variables/complex-with-var-reference/output.txt
 create mode 100644 acceptance/bundle/variables/complex-with-var-reference/script

diff --git a/acceptance/bundle/variables/complex-with-var-reference/databricks.yml b/acceptance/bundle/variables/complex-with-var-reference/databricks.yml
new file mode 100644
index 000000000..104f9a470
--- /dev/null
+++ b/acceptance/bundle/variables/complex-with-var-reference/databricks.yml
@@ -0,0 +1,17 @@
+bundle:
+  name: TestResolveComplexVariableWithVarReference
+
+variables:
+  package_version:
+    default: "1.0.0"
+  cluster_libraries:
+    type: "complex"
+    default:
+      - pypi:
+          package: "cicd_template==${var.package_version}"
+
+resources:
+  jobs:
+    job1:
+      tasks:
+        - libraries: ${var.cluster_libraries}
diff --git a/acceptance/bundle/variables/complex-with-var-reference/output.txt b/acceptance/bundle/variables/complex-with-var-reference/output.txt
new file mode 100644
index 000000000..a5b792ac4
--- /dev/null
+++ b/acceptance/bundle/variables/complex-with-var-reference/output.txt
@@ -0,0 +1,12 @@
+[
+  {
+    "libraries": [
+      {
+        "pypi": {
+          "package": "cicd_template==1.0.0"
+        }
+      }
+    ],
+    "task_key": ""
+  }
+]
diff --git a/acceptance/bundle/variables/complex-with-var-reference/script b/acceptance/bundle/variables/complex-with-var-reference/script
new file mode 100644
index 000000000..0f7353ad1
--- /dev/null
+++ b/acceptance/bundle/variables/complex-with-var-reference/script
@@ -0,0 +1 @@
+$CLI bundle validate -o json | jq .resources.jobs.job1.tasks
diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go
index 4bae7867a..44f6c8dbb 100644
--- a/bundle/config/mutator/resolve_variable_references_test.go
+++ b/bundle/config/mutator/resolve_variable_references_test.go
@@ -7,79 +7,10 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/resources"
-	"github.com/databricks/cli/bundle/config/variable"
-	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/dyn"
-	"github.com/databricks/databricks-sdk-go/service/compute"
-	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/databricks/databricks-sdk-go/service/pipelines"
 	"github.com/stretchr/testify/require"
 )
 
-func TestResolveComplexVariableWithVarReference(t *testing.T) {
-	b := &bundle.Bundle{
-		Config: config.Root{
-			Bundle: config.Bundle{
-				Name: "example",
-			},
-			Variables: map[string]*variable.Variable{
-				"package_version": {
-					Value: "1.0.0",
-				},
-				"cluster_libraries": {
-					Value: [](map[string]any){
-						{
-							"pypi": map[string]string{
-								"package": "cicd_template==${var.package_version}",
-							},
-						},
-					},
-					Type: variable.VariableTypeComplex,
-				},
-			},
-
-			Resources: config.Resources{
-				Jobs: map[string]*resources.Job{
-					"job1": {
-						JobSettings: &jobs.JobSettings{
-							Tasks: []jobs.Task{
-								{
-									Libraries: []compute.Library{},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-
-	ctx := context.Background()
-
-	// Assign the variables to the dynamic configuration.
-	diags := bundle.ApplyFunc(ctx, b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-		err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
-			var p dyn.Path
-			var err error
-
-			p = dyn.MustPathFromString("resources.jobs.job1.tasks[0]")
-			v, err = dyn.SetByPath(v, p.Append(dyn.Key("libraries")), dyn.V("${var.cluster_libraries}"))
-			require.NoError(t, err)
-
-			return v, nil
-		})
-		return diag.FromErr(err)
-	})
-	require.NoError(t, diags.Error())
-
-	diags = bundle.Apply(ctx, b, bundle.Seq(
-		ResolveVariableReferencesInComplexVariables(),
-		ResolveVariableReferences("bundle", "workspace", "variables"),
-	))
-	require.NoError(t, diags.Error())
-	require.Equal(t, "cicd_template==1.0.0", b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].Libraries[0].Pypi.Package)
-}
-
 func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) {
 	testCases := []struct {
 		enabled bool

From 30dec5978131021efc7467fb1c06cb90d3709083 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 15 Jan 2025 18:03:43 +0100
Subject: [PATCH 089/247] Improve resolution of complex variables within
 complex variables (#2157)

## Changes
- Remove ResolveVariableReferencesInComplexVariables - it blocked
complex-within-complex for no good reason.
- Repeat regular resolution twice, it helps with a couple test cases we
have.

There may be a case for running it 3 times or more in a loop, but there
is no test case for that, so this PR is simple incremental improvement.

## Tests
Existing acceptance tests. Previously all unit tests for complex
variables were converted to acceptance tests, to capture this change and
ensure nothing breaks.
---
 .../variables/complex-transitive/output.txt   |  2 +-
 .../complex-within-complex/output.txt         | 17 ++++---
 .../mutator/resolve_variable_references.go    | 45 -------------------
 bundle/phases/initialize.go                   |  6 ++-
 4 files changed, 14 insertions(+), 56 deletions(-)

diff --git a/acceptance/bundle/variables/complex-transitive/output.txt b/acceptance/bundle/variables/complex-transitive/output.txt
index a031e0497..29c41cda5 100644
--- a/acceptance/bundle/variables/complex-transitive/output.txt
+++ b/acceptance/bundle/variables/complex-transitive/output.txt
@@ -1,3 +1,3 @@
 {
-  "spark.databricks.sql.initial.catalog.name": "${var.catalog}"
+  "spark.databricks.sql.initial.catalog.name": "hive_metastore"
 }
diff --git a/acceptance/bundle/variables/complex-within-complex/output.txt b/acceptance/bundle/variables/complex-within-complex/output.txt
index 124bc3363..72e6ef69a 100644
--- a/acceptance/bundle/variables/complex-within-complex/output.txt
+++ b/acceptance/bundle/variables/complex-within-complex/output.txt
@@ -2,17 +2,16 @@ Warning: unknown field: node_type_id
   at resources.jobs.job1.job_clusters[0]
   in databricks.yml:25:11
 
-Error: complex variables cannot contain references to another complex variables
-
 [
   {
     "job_cluster_key": "my_cluster",
-    "new_cluster": null
-  },
-  {
-    "job_cluster_key": "my_cluster",
-    "new_cluster": "${var.cluster}"
+    "new_cluster": {
+      "node_type_id": "Standard_DS3_v2",
+      "num_workers": 2,
+      "spark_conf": {
+        "spark.executor.cores": "2",
+        "spark.executor.memory": "4g"
+      }
+    }
   }
 ]
-
-Exit code: 1
diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go
index 7ad3dfd8d..11ac529d0 100644
--- a/bundle/config/mutator/resolve_variable_references.go
+++ b/bundle/config/mutator/resolve_variable_references.go
@@ -32,19 +32,6 @@ func ResolveVariableReferencesInLookup() bundle.Mutator {
 	}, pattern: dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("lookup")), lookupFn: lookupForVariables}
 }
 
-func ResolveVariableReferencesInComplexVariables() bundle.Mutator {
-	return &resolveVariableReferences{
-		prefixes: []string{
-			"bundle",
-			"workspace",
-			"variables",
-		},
-		pattern:  dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("value")),
-		lookupFn: lookupForComplexVariables,
-		skipFn:   skipResolvingInNonComplexVariables,
-	}
-}
-
 func lookup(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) {
 	if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
 		if path.String() == "workspace.file_path" {
@@ -57,38 +44,6 @@ func lookup(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) {
 	return dyn.GetByPath(v, path)
 }
 
-func lookupForComplexVariables(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) {
-	if path[0].Key() != "variables" {
-		return lookup(v, path, b)
-	}
-
-	varV, err := dyn.GetByPath(v, path[:len(path)-1])
-	if err != nil {
-		return dyn.InvalidValue, err
-	}
-
-	var vv variable.Variable
-	err = convert.ToTyped(&vv, varV)
-	if err != nil {
-		return dyn.InvalidValue, err
-	}
-
-	if vv.Type == variable.VariableTypeComplex {
-		return dyn.InvalidValue, errors.New("complex variables cannot contain references to another complex variables")
-	}
-
-	return lookup(v, path, b)
-}
-
-func skipResolvingInNonComplexVariables(v dyn.Value) bool {
-	switch v.Kind() {
-	case dyn.KindMap, dyn.KindSequence:
-		return false
-	default:
-		return true
-	}
-}
-
 func lookupForVariables(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) {
 	if path[0].Key() != "variables" {
 		return lookup(v, path, b)
diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go
index 50df5634a..0328cc2ff 100644
--- a/bundle/phases/initialize.go
+++ b/bundle/phases/initialize.go
@@ -61,7 +61,11 @@ func Initialize() bundle.Mutator {
 			pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseApplyMutators),
 			mutator.ResolveVariableReferencesInLookup(),
 			mutator.ResolveResourceReferences(),
-			mutator.ResolveVariableReferencesInComplexVariables(),
+			mutator.ResolveVariableReferences(
+				"bundle",
+				"workspace",
+				"variables",
+			),
 			mutator.ResolveVariableReferences(
 				"bundle",
 				"workspace",

From a002a24e4133cbc033f0805cef1c75dda325c9d0 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Wed, 15 Jan 2025 18:22:45 +0100
Subject: [PATCH 090/247] Add a unique schema for recreate pipeline test
 (#2159)

## Changes
Pipeline backend requires `target` to be always specified.

In order to do this we will create an unique schema as part of
`TestBundlePipelineRecreateWithoutAutoApprove` test which will be used
in pipelines

## Tests
```
    helpers_test.go:148: stderr: Destroy complete!
--- PASS: TestBundlePipelineRecreateWithoutAutoApprove (415.39s)
PASS
coverage: [no statements]
ok      github.com/databricks/cli/integration/bundle    416.141s        coverage: [no statements]
```
---
 .../recreate_pipeline/template/databricks.yml.tmpl        | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/integration/bundle/bundles/recreate_pipeline/template/databricks.yml.tmpl b/integration/bundle/bundles/recreate_pipeline/template/databricks.yml.tmpl
index 4ebeb2655..4ea687cf1 100644
--- a/integration/bundle/bundles/recreate_pipeline/template/databricks.yml.tmpl
+++ b/integration/bundle/bundles/recreate_pipeline/template/databricks.yml.tmpl
@@ -9,7 +9,6 @@ variables:
     description: The catalog the DLT pipeline should use.
     default: main
 
-
 resources:
   pipelines:
     foo:
@@ -19,6 +18,13 @@ resources:
             path: ./nb.sql
       development: true
       catalog: ${var.catalog}
+      target: ${resources.schemas.bar.id}
+
+  schemas:
+    bar:
+      name: test-schema-{{.unique_id}}
+      catalog_name: ${var.catalog}
+      comment: This schema was created from DABs
 
 include:
   - "*.yml"

From b273dc5942d7689c76a4ea7ce499cf0d7a26a83d Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 16 Jan 2025 12:20:50 +0100
Subject: [PATCH 091/247] Enable linter 'copyloopvar' and fix the issues
 (#2160)

## Changes
- Remove all unnecessary copies of the loop variable, it is not
necessary since Go 1.22 https://go.dev/blog/loopvar-preview
- Enable the linter that catches this issue
https://github.com/karamaru-alpha/copyloopvar

## Tests
Existing tests.
---
 .golangci.yaml                                |  2 +
 .../config/validate/validate_sync_patterns.go |  8 +-
 cmd/bundle/generate/utils.go                  |  4 +-
 integration/cmd/fs/cat_test.go                | 18 ++--
 integration/cmd/fs/cp_test.go                 | 86 +++++++------------
 integration/cmd/fs/ls_test.go                 | 30 +++----
 integration/cmd/fs/mkdir_test.go              | 18 ++--
 integration/cmd/fs/rm_test.go                 | 30 +++----
 integration/libs/filer/filer_test.go          | 12 +--
 integration/libs/locker/locker_test.go        |  3 +-
 libs/filer/files_client.go                    |  2 -
 11 files changed, 74 insertions(+), 139 deletions(-)

diff --git a/.golangci.yaml b/.golangci.yaml
index 9711a70af..ea6d65db1 100644
--- a/.golangci.yaml
+++ b/.golangci.yaml
@@ -42,6 +42,8 @@ linters-settings:
     disable:
       # good check, but we have too many assert.(No)?Errorf? so excluding for now
       - require-error
+  copyloopvar:
+    check-alias: true
 issues:
   exclude-dirs-use-default: false  # recommended by docs https://golangci-lint.run/usage/false-positives/
   max-issues-per-linter: 1000
diff --git a/bundle/config/validate/validate_sync_patterns.go b/bundle/config/validate/validate_sync_patterns.go
index f5787a81d..04acd28ab 100644
--- a/bundle/config/validate/validate_sync_patterns.go
+++ b/bundle/config/validate/validate_sync_patterns.go
@@ -47,15 +47,13 @@ func checkPatterns(patterns []string, path string, rb bundle.ReadOnlyBundle) (di
 	var errs errgroup.Group
 	var diags diag.Diagnostics
 
-	for i, pattern := range patterns {
-		index := i
-		fullPattern := pattern
+	for index, pattern := range patterns {
 		// If the pattern is negated, strip the negation prefix
 		// and check if the pattern matches any files.
 		// Negation in gitignore syntax means "don't look at this path'
 		// So if p matches nothing it's useless negation, but if there are matches,
 		// it means: do not include these files into result set
-		p := strings.TrimPrefix(fullPattern, "!")
+		p := strings.TrimPrefix(pattern, "!")
 		errs.Go(func() error {
 			fs, err := fileset.NewGlobSet(rb.BundleRoot(), []string{p})
 			if err != nil {
@@ -72,7 +70,7 @@ func checkPatterns(patterns []string, path string, rb bundle.ReadOnlyBundle) (di
 				mu.Lock()
 				diags = diags.Append(diag.Diagnostic{
 					Severity:  diag.Warning,
-					Summary:   fmt.Sprintf("Pattern %s does not match any files", fullPattern),
+					Summary:   fmt.Sprintf("Pattern %s does not match any files", pattern),
 					Locations: []dyn.Location{loc.Location()},
 					Paths:     []dyn.Path{loc.Path()},
 				})
diff --git a/cmd/bundle/generate/utils.go b/cmd/bundle/generate/utils.go
index cbea0bfcc..c2c9bbb55 100644
--- a/cmd/bundle/generate/utils.go
+++ b/cmd/bundle/generate/utils.go
@@ -138,9 +138,7 @@ func (n *downloader) FlushToDisk(ctx context.Context, force bool) error {
 	}
 
 	errs, errCtx := errgroup.WithContext(ctx)
-	for k, v := range n.files {
-		targetPath := k
-		filePath := v
+	for targetPath, filePath := range n.files {
 		errs.Go(func() error {
 			reader, err := n.w.Workspace.Download(errCtx, filePath)
 			if err != nil {
diff --git a/integration/cmd/fs/cat_test.go b/integration/cmd/fs/cat_test.go
index 3e964fe6e..14ec8140e 100644
--- a/integration/cmd/fs/cat_test.go
+++ b/integration/cmd/fs/cat_test.go
@@ -18,13 +18,11 @@ func TestFsCat(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range fsTests {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			f, tmpDir := tc.setupFiler(t)
+			f, tmpDir := testCase.setupFiler(t)
 
 			err := f.Write(context.Background(), "hello.txt", strings.NewReader("abcd"), filer.CreateParentDirectories)
 			require.NoError(t, err)
@@ -40,13 +38,11 @@ func TestFsCatOnADir(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range fsTests {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			f, tmpDir := tc.setupFiler(t)
+			f, tmpDir := testCase.setupFiler(t)
 
 			err := f.Mkdir(context.Background(), "dir1")
 			require.NoError(t, err)
@@ -61,13 +57,11 @@ func TestFsCatOnNonExistentFile(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range fsTests {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			_, tmpDir := tc.setupFiler(t)
+			_, tmpDir := testCase.setupFiler(t)
 
 			_, _, err := testcli.RequireErrorRun(t, ctx, "fs", "cat", path.Join(tmpDir, "non-existent-file"))
 			assert.ErrorIs(t, err, fs.ErrNotExist)
diff --git a/integration/cmd/fs/cp_test.go b/integration/cmd/fs/cp_test.go
index 76aef7acf..6d0266555 100644
--- a/integration/cmd/fs/cp_test.go
+++ b/integration/cmd/fs/cp_test.go
@@ -126,14 +126,12 @@ func TestFsCpDir(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range copyTests() {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			sourceFiler, sourceDir := tc.setupSource(t)
-			targetFiler, targetDir := tc.setupTarget(t)
+			sourceFiler, sourceDir := testCase.setupSource(t)
+			targetFiler, targetDir := testCase.setupTarget(t)
 			setupSourceDir(t, context.Background(), sourceFiler)
 
 			testcli.RequireSuccessfulRun(t, ctx, "fs", "cp", sourceDir, targetDir, "--recursive")
@@ -147,14 +145,12 @@ func TestFsCpFileToFile(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range copyTests() {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			sourceFiler, sourceDir := tc.setupSource(t)
-			targetFiler, targetDir := tc.setupTarget(t)
+			sourceFiler, sourceDir := testCase.setupSource(t)
+			targetFiler, targetDir := testCase.setupTarget(t)
 			setupSourceFile(t, context.Background(), sourceFiler)
 
 			testcli.RequireSuccessfulRun(t, ctx, "fs", "cp", path.Join(sourceDir, "foo.txt"), path.Join(targetDir, "bar.txt"))
@@ -168,14 +164,12 @@ func TestFsCpFileToDir(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range copyTests() {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			sourceFiler, sourceDir := tc.setupSource(t)
-			targetFiler, targetDir := tc.setupTarget(t)
+			sourceFiler, sourceDir := testCase.setupSource(t)
+			targetFiler, targetDir := testCase.setupTarget(t)
 			setupSourceFile(t, context.Background(), sourceFiler)
 
 			testcli.RequireSuccessfulRun(t, ctx, "fs", "cp", path.Join(sourceDir, "foo.txt"), targetDir)
@@ -205,14 +199,12 @@ func TestFsCpDirToDirFileNotOverwritten(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range copyTests() {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			sourceFiler, sourceDir := tc.setupSource(t)
-			targetFiler, targetDir := tc.setupTarget(t)
+			sourceFiler, sourceDir := testCase.setupSource(t)
+			targetFiler, targetDir := testCase.setupTarget(t)
 			setupSourceDir(t, context.Background(), sourceFiler)
 
 			// Write a conflicting file to target
@@ -231,14 +223,12 @@ func TestFsCpFileToDirFileNotOverwritten(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range copyTests() {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			sourceFiler, sourceDir := tc.setupSource(t)
-			targetFiler, targetDir := tc.setupTarget(t)
+			sourceFiler, sourceDir := testCase.setupSource(t)
+			targetFiler, targetDir := testCase.setupTarget(t)
 			setupSourceDir(t, context.Background(), sourceFiler)
 
 			// Write a conflicting file to target
@@ -255,14 +245,12 @@ func TestFsCpFileToFileFileNotOverwritten(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range copyTests() {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			sourceFiler, sourceDir := tc.setupSource(t)
-			targetFiler, targetDir := tc.setupTarget(t)
+			sourceFiler, sourceDir := testCase.setupSource(t)
+			targetFiler, targetDir := testCase.setupTarget(t)
 			setupSourceDir(t, context.Background(), sourceFiler)
 
 			// Write a conflicting file to target
@@ -279,14 +267,12 @@ func TestFsCpDirToDirWithOverwriteFlag(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range copyTests() {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			sourceFiler, sourceDir := tc.setupSource(t)
-			targetFiler, targetDir := tc.setupTarget(t)
+			sourceFiler, sourceDir := testCase.setupSource(t)
+			targetFiler, targetDir := testCase.setupTarget(t)
 			setupSourceDir(t, context.Background(), sourceFiler)
 
 			// Write a conflicting file to target
@@ -303,14 +289,12 @@ func TestFsCpFileToFileWithOverwriteFlag(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range copyTests() {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			sourceFiler, sourceDir := tc.setupSource(t)
-			targetFiler, targetDir := tc.setupTarget(t)
+			sourceFiler, sourceDir := testCase.setupSource(t)
+			targetFiler, targetDir := testCase.setupTarget(t)
 			setupSourceDir(t, context.Background(), sourceFiler)
 
 			// Write a conflicting file to target
@@ -327,14 +311,12 @@ func TestFsCpFileToDirWithOverwriteFlag(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range copyTests() {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			sourceFiler, sourceDir := tc.setupSource(t)
-			targetFiler, targetDir := tc.setupTarget(t)
+			sourceFiler, sourceDir := testCase.setupSource(t)
+			targetFiler, targetDir := testCase.setupTarget(t)
 			setupSourceDir(t, context.Background(), sourceFiler)
 
 			// Write a conflicting file to target
@@ -351,13 +333,11 @@ func TestFsCpErrorsWhenSourceIsDirWithoutRecursiveFlag(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range fsTests {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			_, tmpDir := tc.setupFiler(t)
+			_, tmpDir := testCase.setupFiler(t)
 
 			_, _, err := testcli.RequireErrorRun(t, ctx, "fs", "cp", path.Join(tmpDir), path.Join(tmpDir, "foobar"))
 			r := regexp.MustCompile("source path .* is a directory. Please specify the --recursive flag")
@@ -376,14 +356,12 @@ func TestFsCpSourceIsDirectoryButTargetIsFile(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range copyTests() {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			sourceFiler, sourceDir := tc.setupSource(t)
-			targetFiler, targetDir := tc.setupTarget(t)
+			sourceFiler, sourceDir := testCase.setupSource(t)
+			targetFiler, targetDir := testCase.setupTarget(t)
 			setupSourceDir(t, context.Background(), sourceFiler)
 
 			// Write a conflicting file to target
diff --git a/integration/cmd/fs/ls_test.go b/integration/cmd/fs/ls_test.go
index 25929fdf3..0f53193bf 100644
--- a/integration/cmd/fs/ls_test.go
+++ b/integration/cmd/fs/ls_test.go
@@ -43,13 +43,11 @@ func TestFsLs(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range fsTests {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			f, tmpDir := tc.setupFiler(t)
+			f, tmpDir := testCase.setupFiler(t)
 			setupLsFiles(t, f)
 
 			stdout, stderr := testcli.RequireSuccessfulRun(t, ctx, "fs", "ls", tmpDir, "--output=json")
@@ -77,13 +75,11 @@ func TestFsLsWithAbsolutePaths(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range fsTests {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			f, tmpDir := tc.setupFiler(t)
+			f, tmpDir := testCase.setupFiler(t)
 			setupLsFiles(t, f)
 
 			stdout, stderr := testcli.RequireSuccessfulRun(t, ctx, "fs", "ls", tmpDir, "--output=json", "--absolute")
@@ -111,13 +107,11 @@ func TestFsLsOnFile(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range fsTests {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			f, tmpDir := tc.setupFiler(t)
+			f, tmpDir := testCase.setupFiler(t)
 			setupLsFiles(t, f)
 
 			_, _, err := testcli.RequireErrorRun(t, ctx, "fs", "ls", path.Join(tmpDir, "a", "hello.txt"), "--output=json")
@@ -131,13 +125,11 @@ func TestFsLsOnEmptyDir(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range fsTests {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			_, tmpDir := tc.setupFiler(t)
+			_, tmpDir := testCase.setupFiler(t)
 
 			stdout, stderr := testcli.RequireSuccessfulRun(t, ctx, "fs", "ls", tmpDir, "--output=json")
 			assert.Equal(t, "", stderr.String())
@@ -155,13 +147,11 @@ func TestFsLsForNonexistingDir(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range fsTests {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			_, tmpDir := tc.setupFiler(t)
+			_, tmpDir := testCase.setupFiler(t)
 
 			_, _, err := testcli.RequireErrorRun(t, ctx, "fs", "ls", path.Join(tmpDir, "nonexistent"), "--output=json")
 			assert.ErrorIs(t, err, fs.ErrNotExist)
diff --git a/integration/cmd/fs/mkdir_test.go b/integration/cmd/fs/mkdir_test.go
index eff0599a7..5cea0599c 100644
--- a/integration/cmd/fs/mkdir_test.go
+++ b/integration/cmd/fs/mkdir_test.go
@@ -17,13 +17,11 @@ func TestFsMkdir(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range fsTests {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			f, tmpDir := tc.setupFiler(t)
+			f, tmpDir := testCase.setupFiler(t)
 
 			// create directory "a"
 			stdout, stderr := testcli.RequireSuccessfulRun(t, ctx, "fs", "mkdir", path.Join(tmpDir, "a"))
@@ -43,13 +41,11 @@ func TestFsMkdirCreatesIntermediateDirectories(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range fsTests {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			f, tmpDir := tc.setupFiler(t)
+			f, tmpDir := testCase.setupFiler(t)
 
 			// create directory "a/b/c"
 			stdout, stderr := testcli.RequireSuccessfulRun(t, ctx, "fs", "mkdir", path.Join(tmpDir, "a", "b", "c"))
@@ -81,13 +77,11 @@ func TestFsMkdirWhenDirectoryAlreadyExists(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range fsTests {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			f, tmpDir := tc.setupFiler(t)
+			f, tmpDir := testCase.setupFiler(t)
 
 			// create directory "a"
 			err := f.Mkdir(context.Background(), "a")
diff --git a/integration/cmd/fs/rm_test.go b/integration/cmd/fs/rm_test.go
index 018c7920e..fc19bb5b5 100644
--- a/integration/cmd/fs/rm_test.go
+++ b/integration/cmd/fs/rm_test.go
@@ -17,14 +17,12 @@ func TestFsRmFile(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range fsTests {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			// Create a file
 			ctx := context.Background()
-			f, tmpDir := tc.setupFiler(t)
+			f, tmpDir := testCase.setupFiler(t)
 			err := f.Write(context.Background(), "hello.txt", strings.NewReader("abcd"), filer.CreateParentDirectories)
 			require.NoError(t, err)
 
@@ -48,14 +46,12 @@ func TestFsRmEmptyDir(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range fsTests {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			// Create a directory
 			ctx := context.Background()
-			f, tmpDir := tc.setupFiler(t)
+			f, tmpDir := testCase.setupFiler(t)
 			err := f.Mkdir(context.Background(), "a")
 			require.NoError(t, err)
 
@@ -79,14 +75,12 @@ func TestFsRmNonEmptyDirectory(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range fsTests {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			// Create a directory
 			ctx := context.Background()
-			f, tmpDir := tc.setupFiler(t)
+			f, tmpDir := testCase.setupFiler(t)
 			err := f.Mkdir(context.Background(), "a")
 			require.NoError(t, err)
 
@@ -110,13 +104,11 @@ func TestFsRmForNonExistentFile(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range fsTests {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			_, tmpDir := tc.setupFiler(t)
+			_, tmpDir := testCase.setupFiler(t)
 
 			// Expect error if file does not exist
 			_, _, err := testcli.RequireErrorRun(t, ctx, "fs", "rm", path.Join(tmpDir, "does-not-exist"))
@@ -129,13 +121,11 @@ func TestFsRmDirRecursively(t *testing.T) {
 	t.Parallel()
 
 	for _, testCase := range fsTests {
-		tc := testCase
-
-		t.Run(tc.name, func(t *testing.T) {
+		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
 			ctx := context.Background()
-			f, tmpDir := tc.setupFiler(t)
+			f, tmpDir := testCase.setupFiler(t)
 
 			// Create a directory
 			err := f.Mkdir(context.Background(), "a")
diff --git a/integration/libs/filer/filer_test.go b/integration/libs/filer/filer_test.go
index 21c839e1b..bc1713b30 100644
--- a/integration/libs/filer/filer_test.go
+++ b/integration/libs/filer/filer_test.go
@@ -128,11 +128,9 @@ func TestFilerRecursiveDelete(t *testing.T) {
 		{"files", setupUcVolumesFiler},
 		{"workspace files extensions", setupWsfsExtensionsFiler},
 	} {
-		tc := testCase
-
 		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
-			f, _ := tc.f(t)
+			f, _ := testCase.f(t)
 			ctx := context.Background()
 
 			// Common tests we run across all filers to ensure consistent behavior.
@@ -239,11 +237,9 @@ func TestFilerReadWrite(t *testing.T) {
 		{"files", setupUcVolumesFiler},
 		{"workspace files extensions", setupWsfsExtensionsFiler},
 	} {
-		tc := testCase
-
 		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
-			f, _ := tc.f(t)
+			f, _ := testCase.f(t)
 			ctx := context.Background()
 
 			// Common tests we run across all filers to ensure consistent behavior.
@@ -348,11 +344,9 @@ func TestFilerReadDir(t *testing.T) {
 		{"files", setupUcVolumesFiler},
 		{"workspace files extensions", setupWsfsExtensionsFiler},
 	} {
-		tc := testCase
-
 		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
-			f, _ := tc.f(t)
+			f, _ := testCase.f(t)
 			ctx := context.Background()
 
 			commonFilerReadDirTest(t, ctx, f)
diff --git a/integration/libs/locker/locker_test.go b/integration/libs/locker/locker_test.go
index 524996465..93cb1ffce 100644
--- a/integration/libs/locker/locker_test.go
+++ b/integration/libs/locker/locker_test.go
@@ -66,9 +66,8 @@ func TestLock(t *testing.T) {
 	}
 
 	var wg sync.WaitGroup
-	for i := range numConcurrentLocks {
+	for currentIndex := range numConcurrentLocks {
 		wg.Add(1)
-		currentIndex := i
 		go func() {
 			defer wg.Done()
 			time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go
index 98a534684..88bbadd32 100644
--- a/libs/filer/files_client.go
+++ b/libs/filer/files_client.go
@@ -303,8 +303,6 @@ func (w *FilesClient) recursiveDelete(ctx context.Context, name string) error {
 	group.SetLimit(maxFilesRequestsInFlight)
 
 	for _, file := range filesToDelete {
-		file := file
-
 		// Skip the file if the context has already been cancelled.
 		select {
 		case <-groupCtx.Done():

From 8f34fc796156ada51cebb124d8ae5c19a8ebdd1e Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Thu, 16 Jan 2025 12:39:59 +0100
Subject: [PATCH 092/247] Added output message to warn about slower deployments
 with apps (#2161)

## Changes
When users create one or more Databricks apps in their bundle it can
lead to initial bundle deployment being slower because apps need to wait
until their compute is fully provisioned and started.

This PR adds a message to warn about it. This message will be removed
when `no_compute` option becomes available in TF provider and used in
DABs (https://github.com/databricks/cli/pull/2144)
---
 bundle/apps/slow_deploy_message.go | 29 +++++++++++++++++++++++++++++
 bundle/phases/deploy.go            |  1 +
 2 files changed, 30 insertions(+)
 create mode 100644 bundle/apps/slow_deploy_message.go

diff --git a/bundle/apps/slow_deploy_message.go b/bundle/apps/slow_deploy_message.go
new file mode 100644
index 000000000..6eda39d81
--- /dev/null
+++ b/bundle/apps/slow_deploy_message.go
@@ -0,0 +1,29 @@
+package apps
+
+import (
+	"context"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/diag"
+)
+
+type slowDeployMessage struct{}
+
+// TODO: needs to be removed when when no_compute option becomes available in TF provider and used in DABs
+// See https://github.com/databricks/cli/pull/2144
+func (v *slowDeployMessage) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	if len(b.Config.Resources.Apps) > 0 {
+		cmdio.LogString(ctx, "Databricks apps in your bundle can slow initial deployment as they wait for compute provisioning.")
+	}
+
+	return nil
+}
+
+func (v *slowDeployMessage) Name() string {
+	return "apps.SlowDeployMessage"
+}
+
+func SlowDeployMessage() bundle.Mutator {
+	return &slowDeployMessage{}
+}
diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go
index c6ec04962..1a1ccd47b 100644
--- a/bundle/phases/deploy.go
+++ b/bundle/phases/deploy.go
@@ -136,6 +136,7 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator {
 		bundle.Seq(
 			terraform.StatePush(),
 			terraform.Load(),
+			apps.SlowDeployMessage(),
 			apps.InterpolateVariables(),
 			apps.UploadConfig(),
 			metadata.Compute(),

From 98244606b3358953458fd502c393ae7227094193 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Thu, 16 Jan 2025 13:04:00 +0100
Subject: [PATCH 093/247] Upgrade TF provider to 1.63.0 (#2162)

## Changes
No significant changes to call out for DABs.
---
 bundle/internal/tf/codegen/schema/version.go            | 2 +-
 bundle/internal/tf/schema/resource_external_location.go | 7 +++++++
 bundle/internal/tf/schema/root.go                       | 2 +-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go
index 27c4b16cd..677b8fc10 100644
--- a/bundle/internal/tf/codegen/schema/version.go
+++ b/bundle/internal/tf/codegen/schema/version.go
@@ -1,3 +1,3 @@
 package schema
 
-const ProviderVersion = "1.62.0"
+const ProviderVersion = "1.63.0"
diff --git a/bundle/internal/tf/schema/resource_external_location.go b/bundle/internal/tf/schema/resource_external_location.go
index da28271bc..72411f4dc 100644
--- a/bundle/internal/tf/schema/resource_external_location.go
+++ b/bundle/internal/tf/schema/resource_external_location.go
@@ -13,8 +13,13 @@ type ResourceExternalLocationEncryptionDetails struct {
 
 type ResourceExternalLocation struct {
 	AccessPoint       string                                     `json:"access_point,omitempty"`
+	BrowseOnly        bool                                       `json:"browse_only,omitempty"`
 	Comment           string                                     `json:"comment,omitempty"`
+	CreatedAt         int                                        `json:"created_at,omitempty"`
+	CreatedBy         string                                     `json:"created_by,omitempty"`
+	CredentialId      string                                     `json:"credential_id,omitempty"`
 	CredentialName    string                                     `json:"credential_name"`
+	Fallback          bool                                       `json:"fallback,omitempty"`
 	ForceDestroy      bool                                       `json:"force_destroy,omitempty"`
 	ForceUpdate       bool                                       `json:"force_update,omitempty"`
 	Id                string                                     `json:"id,omitempty"`
@@ -24,6 +29,8 @@ type ResourceExternalLocation struct {
 	Owner             string                                     `json:"owner,omitempty"`
 	ReadOnly          bool                                       `json:"read_only,omitempty"`
 	SkipValidation    bool                                       `json:"skip_validation,omitempty"`
+	UpdatedAt         int                                        `json:"updated_at,omitempty"`
+	UpdatedBy         string                                     `json:"updated_by,omitempty"`
 	Url               string                                     `json:"url"`
 	EncryptionDetails *ResourceExternalLocationEncryptionDetails `json:"encryption_details,omitempty"`
 }
diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go
index 1f89dc64d..7dd3f9210 100644
--- a/bundle/internal/tf/schema/root.go
+++ b/bundle/internal/tf/schema/root.go
@@ -21,7 +21,7 @@ type Root struct {
 
 const ProviderHost = "registry.terraform.io"
 const ProviderSource = "databricks/databricks"
-const ProviderVersion = "1.62.0"
+const ProviderVersion = "1.63.0"
 
 func NewRoot() *Root {
 	return &Root{

From bc1610f6e6ffe329f312492a8dc35ca49cb9a8f3 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 16 Jan 2025 13:14:00 +0100
Subject: [PATCH 094/247] Add a test for complex variable resolution with 3
 levels (#2163)

Follow up to #2157. That PR repeated variable resolution. This test
still does not resolve fully but would resolve with 3 passes. This is
slightly different from complex-transitive-deeper - this test does not
show any errors, the issue is purely not enough passes.
---
 .../complex-transitive-deep/databricks.yml    | 21 +++++++++++++++++++
 .../complex-transitive-deep/output.txt        |  3 +++
 .../variables/complex-transitive-deep/script  |  2 ++
 3 files changed, 26 insertions(+)
 create mode 100644 acceptance/bundle/variables/complex-transitive-deep/databricks.yml
 create mode 100644 acceptance/bundle/variables/complex-transitive-deep/output.txt
 create mode 100644 acceptance/bundle/variables/complex-transitive-deep/script

diff --git a/acceptance/bundle/variables/complex-transitive-deep/databricks.yml b/acceptance/bundle/variables/complex-transitive-deep/databricks.yml
new file mode 100644
index 000000000..1357c291a
--- /dev/null
+++ b/acceptance/bundle/variables/complex-transitive-deep/databricks.yml
@@ -0,0 +1,21 @@
+bundle:
+  name: complex-transitive
+
+variables:
+  catalog:
+    default: hive_metastore
+  spark_conf_1:
+    default:
+      "spark.databricks.sql.initial.catalog.name": ${var.catalog}
+  spark_conf:
+    default: ${var.spark_conf_1}
+  etl_cluster_config:
+    type: complex
+    default:
+      spark_version: 14.3.x-scala2.12
+      runtime_engine: PHOTON
+      spark_conf: ${var.spark_conf}
+
+resources:
+  clusters:
+    my_cluster: ${var.etl_cluster_config}
diff --git a/acceptance/bundle/variables/complex-transitive-deep/output.txt b/acceptance/bundle/variables/complex-transitive-deep/output.txt
new file mode 100644
index 000000000..a031e0497
--- /dev/null
+++ b/acceptance/bundle/variables/complex-transitive-deep/output.txt
@@ -0,0 +1,3 @@
+{
+  "spark.databricks.sql.initial.catalog.name": "${var.catalog}"
+}
diff --git a/acceptance/bundle/variables/complex-transitive-deep/script b/acceptance/bundle/variables/complex-transitive-deep/script
new file mode 100644
index 000000000..52bb08ed4
--- /dev/null
+++ b/acceptance/bundle/variables/complex-transitive-deep/script
@@ -0,0 +1,2 @@
+# Currently, this incorrectly outputs variable reference instead of resolved value
+$CLI bundle validate -o json | jq '.resources.clusters.my_cluster.spark_conf'

From fa87f22706e2232a33d3a6acd6f3d352579dafad Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Thu, 16 Jan 2025 14:03:35 +0100
Subject: [PATCH 095/247] Changed warning message for apps (#2165)

## Changes
Changed warning message for apps

Original warning message added here:
https://github.com/databricks/cli/pull/2161
---
 bundle/apps/slow_deploy_message.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bundle/apps/slow_deploy_message.go b/bundle/apps/slow_deploy_message.go
index 6eda39d81..87275980a 100644
--- a/bundle/apps/slow_deploy_message.go
+++ b/bundle/apps/slow_deploy_message.go
@@ -14,7 +14,7 @@ type slowDeployMessage struct{}
 // See https://github.com/databricks/cli/pull/2144
 func (v *slowDeployMessage) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	if len(b.Config.Resources.Apps) > 0 {
-		cmdio.LogString(ctx, "Databricks apps in your bundle can slow initial deployment as they wait for compute provisioning.")
+		cmdio.LogString(ctx, "Note: Databricks apps included in this bundle may increase initial deployment time due to compute provisioning.")
 	}
 
 	return nil

From f2bba632cb8ab7649e5a80abd2aa106d631060a7 Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Thu, 16 Jan 2025 18:57:00 +0530
Subject: [PATCH 096/247] Patch references to UC schemas to capture
 dependencies automatically (#1989)

## Changes
Fixes https://github.com/databricks/cli/issues/1977.

This PR modifies the bundle configuration to capture the dependency that
a UC Volume or a DLT pipeline might have on a UC schema at deployment
time. It does so by replacing the schema name with a reference of the
form `${resources.schemas.foo.name}`.

For example:
The following UC Volume definition depends on the UC schema with the
name `schema_name`. This mutator converts this configuration

from:
```
resources:
  volumes:
    bar:
      catalog_name: catalog_name
      name: volume_name
      schema_name: schema_name

  schemas:
    foo:
      catalog_name: catalog_name
      name: schema_name
```

to:

```
resources:
  volumes:
    bar:
      catalog_name: catalog_name
      name: volume_name
      schema_name: ${resources.schemas.foo.name}`

  schemas:
    foo:
      catalog_name: catalog_name
      name: schema_name
```


## Tests
Unit tests and manually.
---
 .../mutator/capture_schema_dependency.go      | 100 +++++++
 .../mutator/capture_schema_dependency_test.go | 277 ++++++++++++++++++
 bundle/phases/initialize.go                   |   2 +
 3 files changed, 379 insertions(+)
 create mode 100644 bundle/config/mutator/capture_schema_dependency.go
 create mode 100644 bundle/config/mutator/capture_schema_dependency_test.go

diff --git a/bundle/config/mutator/capture_schema_dependency.go b/bundle/config/mutator/capture_schema_dependency.go
new file mode 100644
index 000000000..5025c9a0d
--- /dev/null
+++ b/bundle/config/mutator/capture_schema_dependency.go
@@ -0,0 +1,100 @@
+package mutator
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/libs/diag"
+)
+
+type captureSchemaDependency struct{}
+
+// If a user defines a UC schema in the bundle, they can refer to it in DLT pipelines
+// or UC Volumes using the `${resources.schemas.<schema_key>.name}` syntax. Using this
+// syntax allows TF to capture the deploy time dependency this DLT pipeline or UC Volume
+// has on the schema and deploy changes to the schema before deploying the pipeline or volume.
+//
+// This mutator translates any implicit schema references in DLT pipelines or UC Volumes
+// to the explicit syntax.
+func CaptureSchemaDependency() bundle.Mutator {
+	return &captureSchemaDependency{}
+}
+
+func (m *captureSchemaDependency) Name() string {
+	return "CaptureSchemaDependency"
+}
+
+func schemaNameRef(key string) string {
+	return fmt.Sprintf("${resources.schemas.%s.name}", key)
+}
+
+func findSchema(b *bundle.Bundle, catalogName, schemaName string) (string, *resources.Schema) {
+	if catalogName == "" || schemaName == "" {
+		return "", nil
+	}
+
+	for k, s := range b.Config.Resources.Schemas {
+		if s != nil && s.CreateSchema != nil && s.CatalogName == catalogName && s.Name == schemaName {
+			return k, s
+		}
+	}
+	return "", nil
+}
+
+func resolveVolume(v *resources.Volume, b *bundle.Bundle) {
+	if v == nil || v.CreateVolumeRequestContent == nil {
+		return
+	}
+	schemaK, schema := findSchema(b, v.CatalogName, v.SchemaName)
+	if schema == nil {
+		return
+	}
+
+	v.SchemaName = schemaNameRef(schemaK)
+}
+
+func resolvePipelineSchema(p *resources.Pipeline, b *bundle.Bundle) {
+	if p == nil || p.PipelineSpec == nil {
+		return
+	}
+	if p.Schema == "" {
+		return
+	}
+	schemaK, schema := findSchema(b, p.Catalog, p.Schema)
+	if schema == nil {
+		return
+	}
+
+	p.Schema = schemaNameRef(schemaK)
+}
+
+func resolvePipelineTarget(p *resources.Pipeline, b *bundle.Bundle) {
+	if p == nil || p.PipelineSpec == nil {
+		return
+	}
+	if p.Target == "" {
+		return
+	}
+	schemaK, schema := findSchema(b, p.Catalog, p.Target)
+	if schema == nil {
+		return
+	}
+	p.Target = schemaNameRef(schemaK)
+}
+
+func (m *captureSchemaDependency) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	for _, p := range b.Config.Resources.Pipelines {
+		// "schema" and "target" have the same semantics in the DLT API but are mutually
+		// exclusive i.e. only one can be set at a time. If schema is set, the pipeline
+		// is in direct publishing mode and can write tables to multiple schemas
+		// (vs target which is limited to a single schema).
+		resolvePipelineTarget(p, b)
+		resolvePipelineSchema(p, b)
+	}
+	for _, v := range b.Config.Resources.Volumes {
+		resolveVolume(v, b)
+	}
+	return nil
+}
diff --git a/bundle/config/mutator/capture_schema_dependency_test.go b/bundle/config/mutator/capture_schema_dependency_test.go
new file mode 100644
index 000000000..0a94e7748
--- /dev/null
+++ b/bundle/config/mutator/capture_schema_dependency_test.go
@@ -0,0 +1,277 @@
+package mutator
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/databricks-sdk-go/service/catalog"
+	"github.com/databricks/databricks-sdk-go/service/pipelines"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCaptureSchemaDependencyForVolume(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Schemas: map[string]*resources.Schema{
+					"schema1": {
+						CreateSchema: &catalog.CreateSchema{
+							CatalogName: "catalog1",
+							Name:        "foobar",
+						},
+					},
+					"schema2": {
+						CreateSchema: &catalog.CreateSchema{
+							CatalogName: "catalog2",
+							Name:        "foobar",
+						},
+					},
+					"schema3": {
+						CreateSchema: &catalog.CreateSchema{
+							CatalogName: "catalog1",
+							Name:        "barfoo",
+						},
+					},
+					"nilschema":   nil,
+					"emptyschema": {},
+				},
+				Volumes: map[string]*resources.Volume{
+					"volume1": {
+						CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{
+							CatalogName: "catalog1",
+							SchemaName:  "foobar",
+						},
+					},
+					"volume2": {
+						CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{
+							CatalogName: "catalog2",
+							SchemaName:  "foobar",
+						},
+					},
+					"volume3": {
+						CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{
+							CatalogName: "catalog1",
+							SchemaName:  "barfoo",
+						},
+					},
+					"volume4": {
+						CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{
+							CatalogName: "catalogX",
+							SchemaName:  "foobar",
+						},
+					},
+					"volume5": {
+						CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{
+							CatalogName: "catalog1",
+							SchemaName:  "schemaX",
+						},
+					},
+					"nilVolume":   nil,
+					"emptyVolume": {},
+				},
+			},
+		},
+	}
+
+	d := bundle.Apply(context.Background(), b, CaptureSchemaDependency())
+	require.Nil(t, d)
+
+	assert.Equal(t, "${resources.schemas.schema1.name}", b.Config.Resources.Volumes["volume1"].CreateVolumeRequestContent.SchemaName)
+	assert.Equal(t, "${resources.schemas.schema2.name}", b.Config.Resources.Volumes["volume2"].CreateVolumeRequestContent.SchemaName)
+	assert.Equal(t, "${resources.schemas.schema3.name}", b.Config.Resources.Volumes["volume3"].CreateVolumeRequestContent.SchemaName)
+	assert.Equal(t, "foobar", b.Config.Resources.Volumes["volume4"].CreateVolumeRequestContent.SchemaName)
+	assert.Equal(t, "schemaX", b.Config.Resources.Volumes["volume5"].CreateVolumeRequestContent.SchemaName)
+
+	assert.Nil(t, b.Config.Resources.Volumes["nilVolume"])
+	assert.Nil(t, b.Config.Resources.Volumes["emptyVolume"].CreateVolumeRequestContent)
+}
+
+func TestCaptureSchemaDependencyForPipelinesWithTarget(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Schemas: map[string]*resources.Schema{
+					"schema1": {
+						CreateSchema: &catalog.CreateSchema{
+							CatalogName: "catalog1",
+							Name:        "foobar",
+						},
+					},
+					"schema2": {
+						CreateSchema: &catalog.CreateSchema{
+							CatalogName: "catalog2",
+							Name:        "foobar",
+						},
+					},
+					"schema3": {
+						CreateSchema: &catalog.CreateSchema{
+							CatalogName: "catalog1",
+							Name:        "barfoo",
+						},
+					},
+					"nilschema":   nil,
+					"emptyschema": {},
+				},
+				Pipelines: map[string]*resources.Pipeline{
+					"pipeline1": {
+						PipelineSpec: &pipelines.PipelineSpec{
+							Catalog: "catalog1",
+							Schema:  "foobar",
+						},
+					},
+					"pipeline2": {
+						PipelineSpec: &pipelines.PipelineSpec{
+							Catalog: "catalog2",
+							Schema:  "foobar",
+						},
+					},
+					"pipeline3": {
+						PipelineSpec: &pipelines.PipelineSpec{
+							Catalog: "catalog1",
+							Schema:  "barfoo",
+						},
+					},
+					"pipeline4": {
+						PipelineSpec: &pipelines.PipelineSpec{
+							Catalog: "catalogX",
+							Schema:  "foobar",
+						},
+					},
+					"pipeline5": {
+						PipelineSpec: &pipelines.PipelineSpec{
+							Catalog: "catalog1",
+							Schema:  "schemaX",
+						},
+					},
+					"pipeline6": {
+						PipelineSpec: &pipelines.PipelineSpec{
+							Catalog: "",
+							Schema:  "foobar",
+						},
+					},
+					"pipeline7": {
+						PipelineSpec: &pipelines.PipelineSpec{
+							Catalog: "",
+							Schema:  "",
+							Name:    "whatever",
+						},
+					},
+					"nilPipeline":   nil,
+					"emptyPipeline": {},
+				},
+			},
+		},
+	}
+
+	d := bundle.Apply(context.Background(), b, CaptureSchemaDependency())
+	require.Nil(t, d)
+
+	assert.Equal(t, "${resources.schemas.schema1.name}", b.Config.Resources.Pipelines["pipeline1"].Schema)
+	assert.Equal(t, "${resources.schemas.schema2.name}", b.Config.Resources.Pipelines["pipeline2"].Schema)
+	assert.Equal(t, "${resources.schemas.schema3.name}", b.Config.Resources.Pipelines["pipeline3"].Schema)
+	assert.Equal(t, "foobar", b.Config.Resources.Pipelines["pipeline4"].Schema)
+	assert.Equal(t, "schemaX", b.Config.Resources.Pipelines["pipeline5"].Schema)
+	assert.Equal(t, "foobar", b.Config.Resources.Pipelines["pipeline6"].Schema)
+	assert.Equal(t, "", b.Config.Resources.Pipelines["pipeline7"].Schema)
+
+	assert.Nil(t, b.Config.Resources.Pipelines["nilPipeline"])
+	assert.Nil(t, b.Config.Resources.Pipelines["emptyPipeline"].PipelineSpec)
+
+	for _, k := range []string{"pipeline1", "pipeline2", "pipeline3", "pipeline4", "pipeline5", "pipeline6", "pipeline7"} {
+		assert.Empty(t, b.Config.Resources.Pipelines[k].Target)
+	}
+}
+
+func TestCaptureSchemaDependencyForPipelinesWithSchema(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Schemas: map[string]*resources.Schema{
+					"schema1": {
+						CreateSchema: &catalog.CreateSchema{
+							CatalogName: "catalog1",
+							Name:        "foobar",
+						},
+					},
+					"schema2": {
+						CreateSchema: &catalog.CreateSchema{
+							CatalogName: "catalog2",
+							Name:        "foobar",
+						},
+					},
+					"schema3": {
+						CreateSchema: &catalog.CreateSchema{
+							CatalogName: "catalog1",
+							Name:        "barfoo",
+						},
+					},
+					"nilschema":   nil,
+					"emptyschema": {},
+				},
+				Pipelines: map[string]*resources.Pipeline{
+					"pipeline1": {
+						PipelineSpec: &pipelines.PipelineSpec{
+							Catalog: "catalog1",
+							Target:  "foobar",
+						},
+					},
+					"pipeline2": {
+						PipelineSpec: &pipelines.PipelineSpec{
+							Catalog: "catalog2",
+							Target:  "foobar",
+						},
+					},
+					"pipeline3": {
+						PipelineSpec: &pipelines.PipelineSpec{
+							Catalog: "catalog1",
+							Target:  "barfoo",
+						},
+					},
+					"pipeline4": {
+						PipelineSpec: &pipelines.PipelineSpec{
+							Catalog: "catalogX",
+							Target:  "foobar",
+						},
+					},
+					"pipeline5": {
+						PipelineSpec: &pipelines.PipelineSpec{
+							Catalog: "catalog1",
+							Target:  "schemaX",
+						},
+					},
+					"pipeline6": {
+						PipelineSpec: &pipelines.PipelineSpec{
+							Catalog: "",
+							Target:  "foobar",
+						},
+					},
+					"pipeline7": {
+						PipelineSpec: &pipelines.PipelineSpec{
+							Catalog: "",
+							Target:  "",
+							Name:    "whatever",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	d := bundle.Apply(context.Background(), b, CaptureSchemaDependency())
+	require.Nil(t, d)
+	assert.Equal(t, "${resources.schemas.schema1.name}", b.Config.Resources.Pipelines["pipeline1"].Target)
+	assert.Equal(t, "${resources.schemas.schema2.name}", b.Config.Resources.Pipelines["pipeline2"].Target)
+	assert.Equal(t, "${resources.schemas.schema3.name}", b.Config.Resources.Pipelines["pipeline3"].Target)
+	assert.Equal(t, "foobar", b.Config.Resources.Pipelines["pipeline4"].Target)
+	assert.Equal(t, "schemaX", b.Config.Resources.Pipelines["pipeline5"].Target)
+	assert.Equal(t, "foobar", b.Config.Resources.Pipelines["pipeline6"].Target)
+	assert.Equal(t, "", b.Config.Resources.Pipelines["pipeline7"].Target)
+
+	for _, k := range []string{"pipeline1", "pipeline2", "pipeline3", "pipeline4", "pipeline5", "pipeline6", "pipeline7"} {
+		assert.Empty(t, b.Config.Resources.Pipelines[k].Schema)
+	}
+}
diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go
index 0328cc2ff..b21b6d8e7 100644
--- a/bundle/phases/initialize.go
+++ b/bundle/phases/initialize.go
@@ -78,6 +78,8 @@ func Initialize() bundle.Mutator {
 			mutator.MergePipelineClusters(),
 			mutator.MergeApps(),
 
+			mutator.CaptureSchemaDependency(),
+
 			// Provide permission config errors & warnings after initializing all variables
 			permissions.PermissionDiagnostics(),
 			mutator.SetRunAs(),

From 2e70558dc18fe4290f62e2a87db613fb9d4827e5 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 16 Jan 2025 15:39:54 +0100
Subject: [PATCH 097/247] Resolve variables in a loop (#2164)

## Changes
- Instead of doing 2 passes on variable resolution, do a loop until
there are no more updates (or we reach count 100).
- Stacked on top of #2163 which is a regression test for this:
acceptance/bundle/variables/complex-transitive-deep

## Tests
Existing tests, new regression tests.

These tests already passed before, added for completeness:
- acceptance/bundle/variables/cycle
- acceptance/bundle/variables/complex-cross-ref
---
 .../complex-cross-ref/databricks.yml          | 12 ++++
 .../variables/complex-cross-ref/output.txt    | 22 +++++++
 .../bundle/variables/complex-cross-ref/script |  1 +
 .../complex-cycle-self/databricks.yml         |  7 ++
 .../variables/complex-cycle-self/output.txt   |  9 +++
 .../variables/complex-cycle-self/script       |  1 +
 .../variables/complex-cycle/databricks.yml    | 10 +++
 .../bundle/variables/complex-cycle/output.txt |  9 +++
 .../bundle/variables/complex-cycle/script     |  1 +
 .../complex-transitive-deep/output.txt        |  2 +-
 .../bundle/variables/cycle/databricks.yml     |  8 +++
 acceptance/bundle/variables/cycle/output.txt  | 14 ++++
 acceptance/bundle/variables/cycle/script      |  1 +
 .../mutator/resolve_variable_references.go    | 66 +++++++++++++++++--
 bundle/phases/initialize.go                   |  5 --
 15 files changed, 156 insertions(+), 12 deletions(-)
 create mode 100644 acceptance/bundle/variables/complex-cross-ref/databricks.yml
 create mode 100644 acceptance/bundle/variables/complex-cross-ref/output.txt
 create mode 100644 acceptance/bundle/variables/complex-cross-ref/script
 create mode 100644 acceptance/bundle/variables/complex-cycle-self/databricks.yml
 create mode 100644 acceptance/bundle/variables/complex-cycle-self/output.txt
 create mode 100644 acceptance/bundle/variables/complex-cycle-self/script
 create mode 100644 acceptance/bundle/variables/complex-cycle/databricks.yml
 create mode 100644 acceptance/bundle/variables/complex-cycle/output.txt
 create mode 100644 acceptance/bundle/variables/complex-cycle/script
 create mode 100644 acceptance/bundle/variables/cycle/databricks.yml
 create mode 100644 acceptance/bundle/variables/cycle/output.txt
 create mode 100644 acceptance/bundle/variables/cycle/script

diff --git a/acceptance/bundle/variables/complex-cross-ref/databricks.yml b/acceptance/bundle/variables/complex-cross-ref/databricks.yml
new file mode 100644
index 000000000..4459f44df
--- /dev/null
+++ b/acceptance/bundle/variables/complex-cross-ref/databricks.yml
@@ -0,0 +1,12 @@
+bundle:
+  name: complex-cross-ref
+
+variables:
+  a:
+    default:
+      a_1: 500
+      a_2: ${var.b.b_2}
+  b:
+    default:
+      b_1: ${var.a.a_1}
+      b_2: 2.5
diff --git a/acceptance/bundle/variables/complex-cross-ref/output.txt b/acceptance/bundle/variables/complex-cross-ref/output.txt
new file mode 100644
index 000000000..f1b624d29
--- /dev/null
+++ b/acceptance/bundle/variables/complex-cross-ref/output.txt
@@ -0,0 +1,22 @@
+{
+  "a": {
+    "default": {
+      "a_1": 500,
+      "a_2": 2.5
+    },
+    "value": {
+      "a_1": 500,
+      "a_2": 2.5
+    }
+  },
+  "b": {
+    "default": {
+      "b_1": 500,
+      "b_2": 2.5
+    },
+    "value": {
+      "b_1": 500,
+      "b_2": 2.5
+    }
+  }
+}
diff --git a/acceptance/bundle/variables/complex-cross-ref/script b/acceptance/bundle/variables/complex-cross-ref/script
new file mode 100644
index 000000000..0e53f237e
--- /dev/null
+++ b/acceptance/bundle/variables/complex-cross-ref/script
@@ -0,0 +1 @@
+$CLI bundle validate -o json | jq .variables
diff --git a/acceptance/bundle/variables/complex-cycle-self/databricks.yml b/acceptance/bundle/variables/complex-cycle-self/databricks.yml
new file mode 100644
index 000000000..bb461795c
--- /dev/null
+++ b/acceptance/bundle/variables/complex-cycle-self/databricks.yml
@@ -0,0 +1,7 @@
+bundle:
+  name: cycle
+
+variables:
+  a:
+    default:
+      hello: ${var.a}
diff --git a/acceptance/bundle/variables/complex-cycle-self/output.txt b/acceptance/bundle/variables/complex-cycle-self/output.txt
new file mode 100644
index 000000000..fa80154ca
--- /dev/null
+++ b/acceptance/bundle/variables/complex-cycle-self/output.txt
@@ -0,0 +1,9 @@
+Warning: Detected unresolved variables after 11 resolution rounds
+
+Name: cycle
+Target: default
+Workspace:
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/cycle/default
+
+Found 1 warning
diff --git a/acceptance/bundle/variables/complex-cycle-self/script b/acceptance/bundle/variables/complex-cycle-self/script
new file mode 100644
index 000000000..72555b332
--- /dev/null
+++ b/acceptance/bundle/variables/complex-cycle-self/script
@@ -0,0 +1 @@
+$CLI bundle validate
diff --git a/acceptance/bundle/variables/complex-cycle/databricks.yml b/acceptance/bundle/variables/complex-cycle/databricks.yml
new file mode 100644
index 000000000..9784a4e25
--- /dev/null
+++ b/acceptance/bundle/variables/complex-cycle/databricks.yml
@@ -0,0 +1,10 @@
+bundle:
+  name: cycle
+
+variables:
+  a:
+    default:
+      hello: ${var.b}
+  b:
+    default:
+      hello: ${var.a}
diff --git a/acceptance/bundle/variables/complex-cycle/output.txt b/acceptance/bundle/variables/complex-cycle/output.txt
new file mode 100644
index 000000000..fa80154ca
--- /dev/null
+++ b/acceptance/bundle/variables/complex-cycle/output.txt
@@ -0,0 +1,9 @@
+Warning: Detected unresolved variables after 11 resolution rounds
+
+Name: cycle
+Target: default
+Workspace:
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/cycle/default
+
+Found 1 warning
diff --git a/acceptance/bundle/variables/complex-cycle/script b/acceptance/bundle/variables/complex-cycle/script
new file mode 100644
index 000000000..72555b332
--- /dev/null
+++ b/acceptance/bundle/variables/complex-cycle/script
@@ -0,0 +1 @@
+$CLI bundle validate
diff --git a/acceptance/bundle/variables/complex-transitive-deep/output.txt b/acceptance/bundle/variables/complex-transitive-deep/output.txt
index a031e0497..29c41cda5 100644
--- a/acceptance/bundle/variables/complex-transitive-deep/output.txt
+++ b/acceptance/bundle/variables/complex-transitive-deep/output.txt
@@ -1,3 +1,3 @@
 {
-  "spark.databricks.sql.initial.catalog.name": "${var.catalog}"
+  "spark.databricks.sql.initial.catalog.name": "hive_metastore"
 }
diff --git a/acceptance/bundle/variables/cycle/databricks.yml b/acceptance/bundle/variables/cycle/databricks.yml
new file mode 100644
index 000000000..b35196671
--- /dev/null
+++ b/acceptance/bundle/variables/cycle/databricks.yml
@@ -0,0 +1,8 @@
+bundle:
+  name: cycle
+
+variables:
+  a:
+    default: ${var.b}
+  b:
+    default: ${var.a}
diff --git a/acceptance/bundle/variables/cycle/output.txt b/acceptance/bundle/variables/cycle/output.txt
new file mode 100644
index 000000000..ea9c95cd4
--- /dev/null
+++ b/acceptance/bundle/variables/cycle/output.txt
@@ -0,0 +1,14 @@
+Error: cycle detected in field resolution: variables.a.default -> var.b -> var.a -> var.b
+
+{
+  "a": {
+    "default": "${var.b}",
+    "value": "${var.b}"
+  },
+  "b": {
+    "default": "${var.a}",
+    "value": "${var.a}"
+  }
+}
+
+Exit code: 1
diff --git a/acceptance/bundle/variables/cycle/script b/acceptance/bundle/variables/cycle/script
new file mode 100644
index 000000000..0e53f237e
--- /dev/null
+++ b/acceptance/bundle/variables/cycle/script
@@ -0,0 +1 @@
+$CLI bundle validate -o json | jq .variables
diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go
index 11ac529d0..9aa93791f 100644
--- a/bundle/config/mutator/resolve_variable_references.go
+++ b/bundle/config/mutator/resolve_variable_references.go
@@ -3,6 +3,7 @@ package mutator
 import (
 	"context"
 	"errors"
+	"fmt"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
@@ -13,15 +14,37 @@ import (
 	"github.com/databricks/cli/libs/dyn/dynvar"
 )
 
+/*
+For pathological cases, output and time grow exponentially.
+
+On my laptop, timings for acceptance/bundle/variables/complex-cycle:
+rounds           time
+
+	 9          0.10s
+	10          0.13s
+	11          0.27s
+	12          0.68s
+	13          1.98s
+	14          6.28s
+	15         21.70s
+	16         78.16s
+*/
+const maxResolutionRounds = 11
+
 type resolveVariableReferences struct {
-	prefixes []string
-	pattern  dyn.Pattern
-	lookupFn func(dyn.Value, dyn.Path, *bundle.Bundle) (dyn.Value, error)
-	skipFn   func(dyn.Value) bool
+	prefixes    []string
+	pattern     dyn.Pattern
+	lookupFn    func(dyn.Value, dyn.Path, *bundle.Bundle) (dyn.Value, error)
+	skipFn      func(dyn.Value) bool
+	extraRounds int
 }
 
 func ResolveVariableReferences(prefixes ...string) bundle.Mutator {
-	return &resolveVariableReferences{prefixes: prefixes, lookupFn: lookup}
+	return &resolveVariableReferences{
+		prefixes:    prefixes,
+		lookupFn:    lookup,
+		extraRounds: maxResolutionRounds - 1,
+	}
 }
 
 func ResolveVariableReferencesInLookup() bundle.Mutator {
@@ -86,7 +109,36 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle)
 	varPath := dyn.NewPath(dyn.Key("var"))
 
 	var diags diag.Diagnostics
+	maxRounds := 1 + m.extraRounds
 
+	for round := range maxRounds {
+		hasUpdates, newDiags := m.resolveOnce(b, prefixes, varPath)
+
+		diags = diags.Extend(newDiags)
+
+		if diags.HasError() {
+			break
+		}
+
+		if !hasUpdates {
+			break
+		}
+
+		if round >= maxRounds-1 {
+			diags = diags.Append(diag.Diagnostic{
+				Severity: diag.Warning,
+				Summary:  fmt.Sprintf("Detected unresolved variables after %d resolution rounds", round+1),
+				// Would be nice to include names of the variables there, but that would complicate things more
+			})
+			break
+		}
+	}
+	return diags
+}
+
+func (m *resolveVariableReferences) resolveOnce(b *bundle.Bundle, prefixes []dyn.Path, varPath dyn.Path) (bool, diag.Diagnostics) {
+	var diags diag.Diagnostics
+	hasUpdates := false
 	err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
 		// Synthesize a copy of the root that has all fields that are present in the type
 		// but not set in the dynamic value set to their corresponding empty value.
@@ -129,6 +181,7 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle)
 						if m.skipFn != nil && m.skipFn(v) {
 							return dyn.InvalidValue, dynvar.ErrSkipResolution
 						}
+						hasUpdates = true
 						return m.lookupFn(normalized, path, b)
 					}
 				}
@@ -149,5 +202,6 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle)
 	if err != nil {
 		diags = diags.Extend(diag.FromErr(err))
 	}
-	return diags
+
+	return hasUpdates, diags
 }
diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go
index b21b6d8e7..c5b875196 100644
--- a/bundle/phases/initialize.go
+++ b/bundle/phases/initialize.go
@@ -66,11 +66,6 @@ func Initialize() bundle.Mutator {
 				"workspace",
 				"variables",
 			),
-			mutator.ResolveVariableReferences(
-				"bundle",
-				"workspace",
-				"variables",
-			),
 
 			mutator.MergeJobClusters(),
 			mutator.MergeJobParameters(),

From 511c8887a8b6b4916cf65e159d8d44a763c677eb Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Thu, 16 Jan 2025 17:25:17 +0100
Subject: [PATCH 098/247] [Release] Release v0.239.0 (#2167)

### New feature announcement

#### Databricks Apps support

You can now manage Databricks Apps using DABs by defining an `app`
resource in your bundle configuration.
For more information see Databricks documentation
https://docs.databricks.com/en/dev-tools/bundles/resources.html#app

#### Referencing complex variables in complex variables

You can now reference complex variables within other complex variables.
For more details see https://github.com/databricks/cli/pull/2157

CLI:
* Filter out system clusters in cluster picker
([#2131](https://github.com/databricks/cli/pull/2131)).
* Add command line flags for fields that are not in the API request body
([#2155](https://github.com/databricks/cli/pull/2155)).

Bundles:
* Added support for Databricks Apps in DABs
([#1928](https://github.com/databricks/cli/pull/1928)).
* Allow artifact path to be located outside the sync root
([#2128](https://github.com/databricks/cli/pull/2128)).
* Retry app deployment if there is an active deployment in progress
([#2153](https://github.com/databricks/cli/pull/2153)).
* Resolve variables in a loop
([#2164](https://github.com/databricks/cli/pull/2164)).
* Improve resolution of complex variables within complex variables
([#2157](https://github.com/databricks/cli/pull/2157)).
* Added output message to warn about slower deployments with apps
([#2161](https://github.com/databricks/cli/pull/2161)).
* Patch references to UC schemas to capture dependencies automatically
([#1989](https://github.com/databricks/cli/pull/1989)).
* Format default-python template
([#2110](https://github.com/databricks/cli/pull/2110)).
* Encourage the use of root_path in production to ensure single
deployment ([#1712](https://github.com/databricks/cli/pull/1712)).
* Log warnings to stderr for "bundle validate -o json"
([#2109](https://github.com/databricks/cli/pull/2109)).

API Changes:
* Changed `databricks account federation-policy update` command with new
required argument order.
* Changed `databricks account service-principal-federation-policy
update` command with new required argument order.

OpenAPI commit 779817ed8d63031f5ea761fbd25ee84f38feec0d (2025-01-08)
Dependency updates:
* Upgrade TF provider to 1.63.0
([#2162](https://github.com/databricks/cli/pull/2162)).
* Bump golangci-lint version to v1.63.4 from v1.63.1
([#2114](https://github.com/databricks/cli/pull/2114)).
* Bump astral-sh/setup-uv from 4 to 5
([#2116](https://github.com/databricks/cli/pull/2116)).
* Bump golang.org/x/oauth2 from 0.24.0 to 0.25.0
([#2080](https://github.com/databricks/cli/pull/2080)).
* Bump github.com/hashicorp/hc-install from 0.9.0 to 0.9.1
([#2079](https://github.com/databricks/cli/pull/2079)).
* Bump golang.org/x/term from 0.27.0 to 0.28.0
([#2078](https://github.com/databricks/cli/pull/2078)).
* Bump github.com/databricks/databricks-sdk-go from 0.54.0 to 0.55.0
([#2126](https://github.com/databricks/cli/pull/2126)).

---------

Co-authored-by: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
---
 CHANGELOG.md | 44 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5b59fa540..53392e5db 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,49 @@
 # Version changelog
 
+## [Release] Release v0.239.0
+
+### New feature announcement
+
+#### Databricks Apps support
+
+You can now manage Databricks Apps using DABs by defining an `app` resource in your bundle configuration.
+For more information see Databricks documentation https://docs.databricks.com/en/dev-tools/bundles/resources.html#app
+
+#### Referencing complex variables in complex variables
+
+You can now reference complex variables within other complex variables.
+For more details see https://github.com/databricks/cli/pull/2157
+
+CLI:
+ * Filter out system clusters in cluster picker ([#2131](https://github.com/databricks/cli/pull/2131)).
+ * Add command line flags for fields that are not in the API request body ([#2155](https://github.com/databricks/cli/pull/2155)).
+
+Bundles:
+ * Added support for Databricks Apps in DABs ([#1928](https://github.com/databricks/cli/pull/1928)).
+ * Allow artifact path to be located outside the sync root ([#2128](https://github.com/databricks/cli/pull/2128)).
+ * Retry app deployment if there is an active deployment in progress ([#2153](https://github.com/databricks/cli/pull/2153)).
+ * Resolve variables in a loop ([#2164](https://github.com/databricks/cli/pull/2164)).
+ * Improve resolution of complex variables within complex variables ([#2157](https://github.com/databricks/cli/pull/2157)).
+ * Added output message to warn about slower deployments with apps ([#2161](https://github.com/databricks/cli/pull/2161)).
+ * Patch references to UC schemas to capture dependencies automatically ([#1989](https://github.com/databricks/cli/pull/1989)).
+ * Format default-python template ([#2110](https://github.com/databricks/cli/pull/2110)).
+ * Encourage the use of root_path in production to ensure single deployment ([#1712](https://github.com/databricks/cli/pull/1712)).
+ * Log warnings to stderr for "bundle validate -o json" ([#2109](https://github.com/databricks/cli/pull/2109)).
+
+API Changes:
+ * Changed `databricks account federation-policy update` command with new required argument order.
+ * Changed `databricks account service-principal-federation-policy update` command with new required argument order.
+
+OpenAPI commit 779817ed8d63031f5ea761fbd25ee84f38feec0d (2025-01-08)
+Dependency updates:
+ * Upgrade TF provider to 1.63.0 ([#2162](https://github.com/databricks/cli/pull/2162)).
+ * Bump golangci-lint version to v1.63.4 from v1.63.1 ([#2114](https://github.com/databricks/cli/pull/2114)).
+ * Bump astral-sh/setup-uv from 4 to 5 ([#2116](https://github.com/databricks/cli/pull/2116)).
+ * Bump golang.org/x/oauth2 from 0.24.0 to 0.25.0 ([#2080](https://github.com/databricks/cli/pull/2080)).
+ * Bump github.com/hashicorp/hc-install from 0.9.0 to 0.9.1 ([#2079](https://github.com/databricks/cli/pull/2079)).
+ * Bump golang.org/x/term from 0.27.0 to 0.28.0 ([#2078](https://github.com/databricks/cli/pull/2078)).
+ * Bump github.com/databricks/databricks-sdk-go from 0.54.0 to 0.55.0 ([#2126](https://github.com/databricks/cli/pull/2126)).
+
 ## [Release] Release v0.238.0
 
 Bundles:

From 2cd0d88bdd5526f2f899833e22ff0b010d627be8 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Fri, 17 Jan 2025 08:38:47 +0100
Subject: [PATCH 099/247] Format Python code with ruff (#2166)

## Changes

The materialized templates included in #2146 include Python code that we
require to be formatted. Instead of running ruff as part of the
testcase, we can enforce that all Python code in the repository is
formatted. It won't be possible to have a passing acceptance test for
template initialization with unformatted code.
---
 .github/workflows/push.yml                    |  6 +++
 .gitignore                                    |  1 +
 Makefile                                      |  3 +-
 acceptance/bin/sort_blocks.py                 |  5 ++-
 .../my_test_code/src/__main__.py              |  6 +--
 .../python_wheel/my_test_code/src/__main__.py |  6 +--
 .../my_test_code/src/__main__.py              |  6 +--
 .../my_test_code/__main__.py                  |  6 +--
 .../my_test_code/__main__.py                  |  6 +--
 .../.databricks/labs/blueprint/lib/install.py |  2 +-
 .../.databricks/labs/blueprint/lib/main.py    | 38 +++++++++++--------
 libs/notebook/testdata/.ruff.toml             |  2 +
 libs/sync/testdata/.ruff.toml                 |  2 +
 13 files changed, 55 insertions(+), 34 deletions(-)
 create mode 100644 libs/notebook/testdata/.ruff.toml
 create mode 100644 libs/sync/testdata/.ruff.toml

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
index d998224a4..d3363b7e3 100644
--- a/.github/workflows/push.yml
+++ b/.github/workflows/push.yml
@@ -60,6 +60,12 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@887a942a15af3a7626099df99e897a18d9e5ab3a # v5.1.0
 
+      - name: Run ruff
+        uses: astral-sh/ruff-action@31a518504640beb4897d0b9f9e50a2a9196e75ba # v3.0.1
+        with:
+          version: "0.9.1"
+          args: "format --check"
+
       - name: Set go env
         run: |
           echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
diff --git a/.gitignore b/.gitignore
index 274c1341a..2060b6bac 100644
--- a/.gitignore
+++ b/.gitignore
@@ -32,3 +32,4 @@ __pycache__
 .vscode/tasks.json
 
 .databricks
+.ruff_cache
diff --git a/Makefile b/Makefile
index 37cf5567c..1ab247406 100644
--- a/Makefile
+++ b/Makefile
@@ -14,6 +14,7 @@ lintcheck:
 # formatting/goimports will not be applied by 'make lint'. However, it will be applied by 'make fmt'.
 # If you need to ensure that formatting & imports are always fixed, do "make fmt lint"
 fmt:
+	ruff format
 	golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix ./...
 
 test:
@@ -44,7 +45,7 @@ snapshot:
 
 vendor:
 	go mod vendor
-  
+
 schema:
 	go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json
 
diff --git a/acceptance/bin/sort_blocks.py b/acceptance/bin/sort_blocks.py
index f50c6f50f..d558f252a 100755
--- a/acceptance/bin/sort_blocks.py
+++ b/acceptance/bin/sort_blocks.py
@@ -4,6 +4,7 @@ Helper to sort blocks in text file. A block is a set of lines separated from oth
 
 This is to workaround non-determinism in the output.
 """
+
 import sys
 
 blocks = []
@@ -11,10 +12,10 @@ blocks = []
 for line in sys.stdin:
     if not line.strip():
         if blocks and blocks[-1]:
-            blocks.append('')
+            blocks.append("")
         continue
     if not blocks:
-        blocks.append('')
+        blocks.append("")
     blocks[-1] += line
 
 blocks.sort()
diff --git a/bundle/tests/python_wheel/environment_key/my_test_code/src/__main__.py b/bundle/tests/python_wheel/environment_key/my_test_code/src/__main__.py
index 73d045afb..ea918ce2d 100644
--- a/bundle/tests/python_wheel/environment_key/my_test_code/src/__main__.py
+++ b/bundle/tests/python_wheel/environment_key/my_test_code/src/__main__.py
@@ -7,10 +7,10 @@ import sys
 
 def main():
     # This method will print the provided arguments
-    print('Hello from my func')
-    print('Got arguments:')
+    print("Hello from my func")
+    print("Got arguments:")
     print(sys.argv)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()
diff --git a/bundle/tests/python_wheel/python_wheel/my_test_code/src/__main__.py b/bundle/tests/python_wheel/python_wheel/my_test_code/src/__main__.py
index 73d045afb..ea918ce2d 100644
--- a/bundle/tests/python_wheel/python_wheel/my_test_code/src/__main__.py
+++ b/bundle/tests/python_wheel/python_wheel/my_test_code/src/__main__.py
@@ -7,10 +7,10 @@ import sys
 
 def main():
     # This method will print the provided arguments
-    print('Hello from my func')
-    print('Got arguments:')
+    print("Hello from my func")
+    print("Got arguments:")
     print(sys.argv)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()
diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py
index 73d045afb..ea918ce2d 100644
--- a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py
+++ b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py
@@ -7,10 +7,10 @@ import sys
 
 def main():
     # This method will print the provided arguments
-    print('Hello from my func')
-    print('Got arguments:')
+    print("Hello from my func")
+    print("Got arguments:")
     print(sys.argv)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()
diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact/my_test_code/__main__.py b/bundle/tests/python_wheel/python_wheel_no_artifact/my_test_code/__main__.py
index 73d045afb..ea918ce2d 100644
--- a/bundle/tests/python_wheel/python_wheel_no_artifact/my_test_code/__main__.py
+++ b/bundle/tests/python_wheel/python_wheel_no_artifact/my_test_code/__main__.py
@@ -7,10 +7,10 @@ import sys
 
 def main():
     # This method will print the provided arguments
-    print('Hello from my func')
-    print('Got arguments:')
+    print("Hello from my func")
+    print("Got arguments:")
     print(sys.argv)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()
diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py
index 73d045afb..ea918ce2d 100644
--- a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py
+++ b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py
@@ -7,10 +7,10 @@ import sys
 
 def main():
     # This method will print the provided arguments
-    print('Hello from my func')
-    print('Got arguments:')
+    print("Hello from my func")
+    print("Got arguments:")
     print(sys.argv)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()
diff --git a/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/install.py b/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/install.py
index 6873257d5..a162da342 100644
--- a/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/install.py
+++ b/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/install.py
@@ -1 +1 @@
-print(f'setting up important infrastructure')
+print(f"setting up important infrastructure")
diff --git a/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/main.py b/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/main.py
index 769ee73ee..e5866d6ae 100644
--- a/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/main.py
+++ b/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/main.py
@@ -2,26 +2,34 @@ import os, sys, json
 
 payload = json.loads(sys.argv[1])
 
-if 'echo' == payload['command']:
-    json.dump({
-        'command': payload['command'],
-        'flags': payload['flags'],
-        'env': {k:v for k,v in os.environ.items()}
-    }, sys.stdout)
+if "echo" == payload["command"]:
+    json.dump(
+        {
+            "command": payload["command"],
+            "flags": payload["flags"],
+            "env": {k: v for k, v in os.environ.items()},
+        },
+        sys.stdout,
+    )
     sys.exit(0)
 
-if 'table' == payload['command']:
+if "table" == payload["command"]:
     sys.stderr.write("some intermediate info\n")
-    json.dump({'records': [
-        {'key': 'First', 'value': 'Second'},
-        {'key': 'Third', 'value': 'Fourth'},
-    ]}, sys.stdout)
+    json.dump(
+        {
+            "records": [
+                {"key": "First", "value": "Second"},
+                {"key": "Third", "value": "Fourth"},
+            ]
+        },
+        sys.stdout,
+    )
     sys.exit(0)
 
-print(f'host is {os.environ["DATABRICKS_HOST"]}')
+print(f"host is {os.environ['DATABRICKS_HOST']}")
 
-print(f'[{payload["command"]}] command flags are {payload["flags"]}')
+print(f"[{payload['command']}] command flags are {payload['flags']}")
 
-answer = input('What is your name? ')
+answer = input("What is your name? ")
 
-print(f'Hello, {answer}!')
+print(f"Hello, {answer}!")
diff --git a/libs/notebook/testdata/.ruff.toml b/libs/notebook/testdata/.ruff.toml
new file mode 100644
index 000000000..43f86042e
--- /dev/null
+++ b/libs/notebook/testdata/.ruff.toml
@@ -0,0 +1,2 @@
+[format]
+exclude = ["*.ipynb"]
diff --git a/libs/sync/testdata/.ruff.toml b/libs/sync/testdata/.ruff.toml
new file mode 100644
index 000000000..43f86042e
--- /dev/null
+++ b/libs/sync/testdata/.ruff.toml
@@ -0,0 +1,2 @@
+[format]
+exclude = ["*.ipynb"]

From 9061635789cc926de19396b0b2ec0dee74ab6ecd Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Fri, 17 Jan 2025 10:38:01 +0100
Subject: [PATCH 100/247] Default to forward slash-separated paths for path
 translation (#2145)

## Changes

This came up in #2122 where relative library paths showed up with
backslashes on Windows. It's hard to run acceptance tests where paths
may be in either form. This change updates path translation logic to
always use forward slash-separated paths, including for absolute paths.

## Tests

* Unit tests pass.
* Confirmed that code where library paths are used uses the `filepath`
package for path manipulation. The functions in this package always
normalize their inputs to be platform-native paths.
* Confirmed that code that uses absolute paths works with forward
slash-separated paths on Windows.
---
 bundle/config/mutator/translate_paths.go      | 38 ++++++++++---------
 .../mutator/translate_paths_artifacts_test.go |  4 +-
 .../translate_paths_dashboards_test.go        |  2 +-
 bundle/config/mutator/translate_paths_test.go | 27 +++++++------
 .../tests/relative_path_with_includes_test.go |  8 ++--
 5 files changed, 41 insertions(+), 38 deletions(-)

diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go
index a2c830be3..1eda578fa 100644
--- a/bundle/config/mutator/translate_paths.go
+++ b/bundle/config/mutator/translate_paths.go
@@ -136,7 +136,7 @@ func (t *translateContext) rewritePath(
 	}
 
 	// Local path is relative to the directory the resource was defined in.
-	localPath := filepath.Join(dir, filepath.FromSlash(input))
+	localPath := filepath.Join(dir, input)
 	if interp, ok := t.seen[localPath]; ok {
 		return interp, nil
 	}
@@ -151,6 +151,10 @@ func (t *translateContext) rewritePath(
 		return "", fmt.Errorf("path %s is not contained in sync root path", localPath)
 	}
 
+	// Normalize paths to separated by forward slashes.
+	localPath = filepath.ToSlash(localPath)
+	localRelPath = filepath.ToSlash(localRelPath)
+
 	// Convert local path into workspace path via specified function.
 	var interp string
 	switch opts.Mode {
@@ -180,9 +184,9 @@ func (t *translateContext) rewritePath(
 }
 
 func (t *translateContext) translateNotebookPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) {
-	nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, filepath.ToSlash(localRelPath))
+	nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, localRelPath)
 	if errors.Is(err, fs.ErrNotExist) {
-		if filepath.Ext(localFullPath) != notebook.ExtensionNone {
+		if path.Ext(localFullPath) != notebook.ExtensionNone {
 			return "", fmt.Errorf("notebook %s not found", literal)
 		}
 
@@ -198,7 +202,7 @@ func (t *translateContext) translateNotebookPath(ctx context.Context, literal, l
 		// way we can provide a more targeted error message.
 		for _, ext := range extensions {
 			literalWithExt := literal + ext
-			localRelPathWithExt := filepath.ToSlash(localRelPath + ext)
+			localRelPathWithExt := localRelPath + ext
 			if _, err := fs.Stat(t.b.SyncRoot, localRelPathWithExt); err == nil {
 				return "", fmt.Errorf(`notebook %s not found. Did you mean %s?
 Local notebook references are expected to contain one of the following
@@ -218,42 +222,42 @@ to contain one of the following file extensions: [%s]`, literal, strings.Join(ex
 	}
 
 	// Upon import, notebooks are stripped of their extension.
-	localRelPathNoExt := strings.TrimSuffix(localRelPath, filepath.Ext(localRelPath))
-	return path.Join(t.remoteRoot, filepath.ToSlash(localRelPathNoExt)), nil
+	localRelPathNoExt := strings.TrimSuffix(localRelPath, path.Ext(localRelPath))
+	return path.Join(t.remoteRoot, localRelPathNoExt), nil
 }
 
 func (t *translateContext) translateFilePath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) {
-	nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, filepath.ToSlash(localRelPath))
+	nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, localRelPath)
 	if errors.Is(err, fs.ErrNotExist) {
 		return "", fmt.Errorf("file %s not found", literal)
 	}
 	if err != nil {
-		return "", fmt.Errorf("unable to determine if %s is not a notebook: %w", localFullPath, err)
+		return "", fmt.Errorf("unable to determine if %s is not a notebook: %w", filepath.FromSlash(localFullPath), err)
 	}
 	if nb {
 		return "", ErrIsNotebook{localFullPath}
 	}
-	return path.Join(t.remoteRoot, filepath.ToSlash(localRelPath)), nil
+	return path.Join(t.remoteRoot, localRelPath), nil
 }
 
 func (t *translateContext) translateDirectoryPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) {
-	info, err := t.b.SyncRoot.Stat(filepath.ToSlash(localRelPath))
+	info, err := t.b.SyncRoot.Stat(localRelPath)
 	if err != nil {
 		return "", err
 	}
 	if !info.IsDir() {
-		return "", fmt.Errorf("%s is not a directory", localFullPath)
+		return "", fmt.Errorf("%s is not a directory", filepath.FromSlash(localFullPath))
 	}
-	return path.Join(t.remoteRoot, filepath.ToSlash(localRelPath)), nil
+	return path.Join(t.remoteRoot, localRelPath), nil
 }
 
 func (t *translateContext) translateLocalAbsoluteFilePath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) {
-	info, err := t.b.SyncRoot.Stat(filepath.ToSlash(localRelPath))
+	info, err := t.b.SyncRoot.Stat(localRelPath)
 	if errors.Is(err, fs.ErrNotExist) {
 		return "", fmt.Errorf("file %s not found", literal)
 	}
 	if err != nil {
-		return "", fmt.Errorf("unable to determine if %s is a file: %w", localFullPath, err)
+		return "", fmt.Errorf("unable to determine if %s is a file: %w", filepath.FromSlash(localFullPath), err)
 	}
 	if info.IsDir() {
 		return "", fmt.Errorf("expected %s to be a file but found a directory", literal)
@@ -262,12 +266,12 @@ func (t *translateContext) translateLocalAbsoluteFilePath(ctx context.Context, l
 }
 
 func (t *translateContext) translateLocalAbsoluteDirectoryPath(ctx context.Context, literal, localFullPath, _ string) (string, error) {
-	info, err := os.Stat(localFullPath)
+	info, err := os.Stat(filepath.FromSlash(localFullPath))
 	if errors.Is(err, fs.ErrNotExist) {
 		return "", fmt.Errorf("directory %s not found", literal)
 	}
 	if err != nil {
-		return "", fmt.Errorf("unable to determine if %s is a directory: %w", localFullPath, err)
+		return "", fmt.Errorf("unable to determine if %s is a directory: %w", filepath.FromSlash(localFullPath), err)
 	}
 	if !info.IsDir() {
 		return "", fmt.Errorf("expected %s to be a directory but found a file", literal)
@@ -281,7 +285,7 @@ func (t *translateContext) translateLocalRelativePath(ctx context.Context, liter
 
 func (t *translateContext) translateLocalRelativeWithPrefixPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) {
 	if !strings.HasPrefix(localRelPath, ".") {
-		localRelPath = "." + string(filepath.Separator) + localRelPath
+		localRelPath = "./" + localRelPath
 	}
 	return localRelPath, nil
 }
diff --git a/bundle/config/mutator/translate_paths_artifacts_test.go b/bundle/config/mutator/translate_paths_artifacts_test.go
index fb402b488..0d1af6156 100644
--- a/bundle/config/mutator/translate_paths_artifacts_test.go
+++ b/bundle/config/mutator/translate_paths_artifacts_test.go
@@ -46,7 +46,7 @@ func TestTranslatePathsArtifacts_InsideSyncRoot(t *testing.T) {
 	require.NoError(t, diags.Error())
 
 	// Assert that the artifact path has been converted to a local absolute path.
-	assert.Equal(t, lib, b.Config.Artifacts["my_artifact"].Path)
+	assert.Equal(t, filepath.ToSlash(lib), b.Config.Artifacts["my_artifact"].Path)
 }
 
 func TestTranslatePathsArtifacts_OutsideSyncRoot(t *testing.T) {
@@ -79,5 +79,5 @@ func TestTranslatePathsArtifacts_OutsideSyncRoot(t *testing.T) {
 	require.NoError(t, diags.Error())
 
 	// Assert that the artifact path has been converted to a local absolute path.
-	assert.Equal(t, lib, b.Config.Artifacts["my_artifact"].Path)
+	assert.Equal(t, filepath.ToSlash(lib), b.Config.Artifacts["my_artifact"].Path)
 }
diff --git a/bundle/config/mutator/translate_paths_dashboards_test.go b/bundle/config/mutator/translate_paths_dashboards_test.go
index 5e4e69f5d..02fba92e0 100644
--- a/bundle/config/mutator/translate_paths_dashboards_test.go
+++ b/bundle/config/mutator/translate_paths_dashboards_test.go
@@ -48,7 +48,7 @@ func TestTranslatePathsDashboards_FilePathRelativeSubDirectory(t *testing.T) {
 	// Assert that the file path for the dashboard has been converted to its local absolute path.
 	assert.Equal(
 		t,
-		filepath.Join(dir, "src", "my_dashboard.lvdash.json"),
+		filepath.ToSlash(filepath.Join(dir, "src", "my_dashboard.lvdash.json")),
 		b.Config.Resources.Dashboards["dashboard"].FilePath,
 	)
 }
diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go
index 493abb8c5..aa6488ab0 100644
--- a/bundle/config/mutator/translate_paths_test.go
+++ b/bundle/config/mutator/translate_paths_test.go
@@ -6,7 +6,6 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
-	"strings"
 	"testing"
 
 	"github.com/databricks/cli/bundle"
@@ -226,7 +225,7 @@ func TestTranslatePaths(t *testing.T) {
 	)
 	assert.Equal(
 		t,
-		filepath.Join("dist", "task.whl"),
+		"dist/task.whl",
 		b.Config.Resources.Jobs["job"].Tasks[0].Libraries[0].Whl,
 	)
 	assert.Equal(
@@ -251,7 +250,7 @@ func TestTranslatePaths(t *testing.T) {
 	)
 	assert.Equal(
 		t,
-		filepath.Join("dist", "task.jar"),
+		"dist/task.jar",
 		b.Config.Resources.Jobs["job"].Tasks[5].Libraries[0].Jar,
 	)
 	assert.Equal(
@@ -362,7 +361,7 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
 	)
 	assert.Equal(
 		t,
-		filepath.Join("job", "dist", "task.jar"),
+		"job/dist/task.jar",
 		b.Config.Resources.Jobs["job"].Tasks[1].Libraries[0].Jar,
 	)
 	assert.Equal(
@@ -774,8 +773,8 @@ func TestTranslatePathJobEnvironments(t *testing.T) {
 	diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
 	require.NoError(t, diags.Error())
 
-	assert.Equal(t, strings.Join([]string{".", "job", "dist", "env1.whl"}, string(os.PathSeparator)), b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[0])
-	assert.Equal(t, strings.Join([]string{".", "dist", "env2.whl"}, string(os.PathSeparator)), b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[1])
+	assert.Equal(t, "./job/dist/env1.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[0])
+	assert.Equal(t, "./dist/env2.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[1])
 	assert.Equal(t, "simplejson", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[2])
 	assert.Equal(t, "/Workspace/Users/foo@bar.com/test.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[3])
 	assert.Equal(t, "--extra-index-url https://name:token@gitlab.com/api/v4/projects/9876/packages/pypi/simple foobar", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[4])
@@ -839,7 +838,7 @@ func TestTranslatePathWithComplexVariables(t *testing.T) {
 
 	assert.Equal(
 		t,
-		filepath.Join("variables", "local", "whl.whl"),
+		"variables/local/whl.whl",
 		b.Config.Resources.Jobs["job"].Tasks[0].Libraries[0].Whl,
 	)
 }
@@ -952,34 +951,34 @@ func TestTranslatePathsWithSourceLinkedDeployment(t *testing.T) {
 	// updated to source path
 	assert.Equal(
 		t,
-		filepath.Join(dir, "my_job_notebook"),
+		dir+"/my_job_notebook",
 		b.Config.Resources.Jobs["job"].Tasks[0].NotebookTask.NotebookPath,
 	)
 	assert.Equal(
 		t,
-		filepath.Join(dir, "requirements.txt"),
+		dir+"/requirements.txt",
 		b.Config.Resources.Jobs["job"].Tasks[2].Libraries[0].Requirements,
 	)
 	assert.Equal(
 		t,
-		filepath.Join(dir, "my_python_file.py"),
+		dir+"/my_python_file.py",
 		b.Config.Resources.Jobs["job"].Tasks[3].SparkPythonTask.PythonFile,
 	)
 	assert.Equal(
 		t,
-		filepath.Join(dir, "my_pipeline_notebook"),
+		dir+"/my_pipeline_notebook",
 		b.Config.Resources.Pipelines["pipeline"].Libraries[0].Notebook.Path,
 	)
 	assert.Equal(
 		t,
-		filepath.Join(dir, "my_python_file.py"),
+		dir+"/my_python_file.py",
 		b.Config.Resources.Pipelines["pipeline"].Libraries[2].File.Path,
 	)
 
 	// left as is
 	assert.Equal(
 		t,
-		filepath.Join("dist", "task.whl"),
+		"dist/task.whl",
 		b.Config.Resources.Jobs["job"].Tasks[0].Libraries[0].Whl,
 	)
 	assert.Equal(
@@ -989,7 +988,7 @@ func TestTranslatePathsWithSourceLinkedDeployment(t *testing.T) {
 	)
 	assert.Equal(
 		t,
-		filepath.Join("dist", "task.jar"),
+		"dist/task.jar",
 		b.Config.Resources.Jobs["job"].Tasks[4].Libraries[0].Jar,
 	)
 	assert.Equal(
diff --git a/bundle/tests/relative_path_with_includes_test.go b/bundle/tests/relative_path_with_includes_test.go
index 8efac0039..7249cac1f 100644
--- a/bundle/tests/relative_path_with_includes_test.go
+++ b/bundle/tests/relative_path_with_includes_test.go
@@ -17,8 +17,8 @@ func TestRelativePathsWithIncludes(t *testing.T) {
 	diags := bundle.Apply(context.Background(), b, m)
 	assert.NoError(t, diags.Error())
 
-	assert.Equal(t, filepath.Join(b.SyncRootPath, "artifact_a"), b.Config.Artifacts["test_a"].Path)
-	assert.Equal(t, filepath.Join(b.SyncRootPath, "subfolder", "artifact_b"), b.Config.Artifacts["test_b"].Path)
+	assert.Equal(t, b.SyncRootPath+"/artifact_a", b.Config.Artifacts["test_a"].Path)
+	assert.Equal(t, b.SyncRootPath+"/subfolder/artifact_b", b.Config.Artifacts["test_b"].Path)
 
 	assert.ElementsMatch(
 		t,
@@ -37,6 +37,6 @@ func TestRelativePathsWithIncludes(t *testing.T) {
 		b.Config.Sync.Exclude,
 	)
 
-	assert.Equal(t, filepath.Join("dist", "job_a.whl"), b.Config.Resources.Jobs["job_a"].Tasks[0].Libraries[0].Whl)
-	assert.Equal(t, filepath.Join("subfolder", "dist", "job_b.whl"), b.Config.Resources.Jobs["job_b"].Tasks[0].Libraries[0].Whl)
+	assert.Equal(t, "dist/job_a.whl", b.Config.Resources.Jobs["job_a"].Tasks[0].Libraries[0].Whl)
+	assert.Equal(t, "subfolder/dist/job_b.whl", b.Config.Resources.Jobs["job_b"].Tasks[0].Libraries[0].Whl)
 }

From 89eb5563181f0c82c69e39cf0f8ee1bb6d942e70 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Fri, 17 Jan 2025 11:22:49 +0100
Subject: [PATCH 101/247] Migrate path translation tests to acceptance tests
 (#2122)

## Changes

The assertions on the output made are now captured in the `output.*`
files. These don't capture intent like actual assertions do, but we
still have regular test coverage in the path translation tests under
`bundle/config/mutator`.

## Tests

Tests pass.
---
 .../bundle/paths}/fallback/README.md          |   0
 .../bundle/paths/fallback}/databricks.yml     |   2 +-
 .../bundle/paths/fallback/output.job.json     |  67 +++++++++++
 .../paths/fallback/output.pipeline.json       |  22 ++++
 acceptance/bundle/paths/fallback/output.txt   |  18 +++
 .../bundle/paths}/fallback/override_job.yml   |   0
 .../paths}/fallback/override_pipeline.yml     |   0
 .../paths}/fallback/resources/my_job.yml      |  12 ++
 .../paths}/fallback/resources/my_pipeline.yml |   0
 acceptance/bundle/paths/fallback/script       |  10 ++
 .../bundle/paths/fallback/script.cleanup      |   1 +
 .../paths}/fallback/src/dbt_project/.gitkeep  |   0
 .../bundle/paths}/fallback/src/file.py        |   0
 .../bundle/paths}/fallback/src/file1.py       |   0
 .../bundle/paths}/fallback/src/file2.py       |   0
 .../bundle/paths}/fallback/src/notebook.py    |   0
 .../bundle/paths}/fallback/src/notebook1.py   |   0
 .../bundle/paths}/fallback/src/notebook2.py   |   0
 .../bundle/paths}/fallback/src/sql.sql        |   0
 .../bundle/paths}/nominal/README.md           |   0
 .../bundle/paths/nominal}/databricks.yml      |   2 +-
 .../bundle/paths/nominal/output.job.json      |  89 ++++++++++++++
 .../bundle/paths/nominal/output.pipeline.json |  22 ++++
 acceptance/bundle/paths/nominal/output.txt    |  18 +++
 .../bundle/paths}/nominal/override_job.yml    |   0
 .../paths}/nominal/override_pipeline.yml      |   0
 .../paths}/nominal/resources/my_job.yml       |  14 +++
 .../paths}/nominal/resources/my_pipeline.yml  |   0
 acceptance/bundle/paths/nominal/script        |  10 ++
 .../bundle/paths/nominal/script.cleanup       |   1 +
 .../paths}/nominal/src/dbt_project/.gitkeep   |   0
 .../bundle/paths}/nominal/src/file.py         |   0
 .../bundle/paths}/nominal/src/file1.py        |   0
 .../bundle/paths}/nominal/src/file2.py        |   0
 .../bundle/paths}/nominal/src/notebook.py     |   0
 .../bundle/paths}/nominal/src/notebook1.py    |   0
 .../bundle/paths}/nominal/src/notebook2.py    |   0
 .../bundle/paths}/nominal/src/sql.sql         |   0
 .../relative_path_translation/databricks.yml  |   0
 .../output.default.json                       |   6 +
 .../output.override.json                      |   6 +
 .../relative_path_translation/output.txt      |   4 +
 .../resources/job.yml                         |   8 ++
 .../paths/relative_path_translation/script    |   4 +
 .../relative_path_translation/src/file1.py    |   0
 .../relative_path_translation/src/file2.py    |   0
 bundle/tests/path_translation_test.go         | 112 ------------------
 .../tests/relative_path_translation_test.go   |  28 -----
 48 files changed, 314 insertions(+), 142 deletions(-)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/fallback/README.md (100%)
 rename {bundle/tests/path_translation/nominal => acceptance/bundle/paths/fallback}/databricks.yml (80%)
 create mode 100644 acceptance/bundle/paths/fallback/output.job.json
 create mode 100644 acceptance/bundle/paths/fallback/output.pipeline.json
 create mode 100644 acceptance/bundle/paths/fallback/output.txt
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/fallback/override_job.yml (100%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/fallback/override_pipeline.yml (100%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/fallback/resources/my_job.yml (71%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/fallback/resources/my_pipeline.yml (100%)
 create mode 100644 acceptance/bundle/paths/fallback/script
 create mode 100644 acceptance/bundle/paths/fallback/script.cleanup
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/fallback/src/dbt_project/.gitkeep (100%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/fallback/src/file.py (100%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/fallback/src/file1.py (100%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/fallback/src/file2.py (100%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/fallback/src/notebook.py (100%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/fallback/src/notebook1.py (100%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/fallback/src/notebook2.py (100%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/fallback/src/sql.sql (100%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/nominal/README.md (100%)
 rename {bundle/tests/path_translation/fallback => acceptance/bundle/paths/nominal}/databricks.yml (79%)
 create mode 100644 acceptance/bundle/paths/nominal/output.job.json
 create mode 100644 acceptance/bundle/paths/nominal/output.pipeline.json
 create mode 100644 acceptance/bundle/paths/nominal/output.txt
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/nominal/override_job.yml (100%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/nominal/override_pipeline.yml (100%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/nominal/resources/my_job.yml (74%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/nominal/resources/my_pipeline.yml (100%)
 create mode 100644 acceptance/bundle/paths/nominal/script
 create mode 100644 acceptance/bundle/paths/nominal/script.cleanup
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/nominal/src/dbt_project/.gitkeep (100%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/nominal/src/file.py (100%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/nominal/src/file1.py (100%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/nominal/src/file2.py (100%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/nominal/src/notebook.py (100%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/nominal/src/notebook1.py (100%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/nominal/src/notebook2.py (100%)
 rename {bundle/tests/path_translation => acceptance/bundle/paths}/nominal/src/sql.sql (100%)
 rename {bundle/tests => acceptance/bundle/paths}/relative_path_translation/databricks.yml (100%)
 create mode 100644 acceptance/bundle/paths/relative_path_translation/output.default.json
 create mode 100644 acceptance/bundle/paths/relative_path_translation/output.override.json
 create mode 100644 acceptance/bundle/paths/relative_path_translation/output.txt
 rename {bundle/tests => acceptance/bundle/paths}/relative_path_translation/resources/job.yml (66%)
 create mode 100644 acceptance/bundle/paths/relative_path_translation/script
 rename {bundle/tests => acceptance/bundle/paths}/relative_path_translation/src/file1.py (100%)
 rename {bundle/tests => acceptance/bundle/paths}/relative_path_translation/src/file2.py (100%)
 delete mode 100644 bundle/tests/path_translation_test.go
 delete mode 100644 bundle/tests/relative_path_translation_test.go

diff --git a/bundle/tests/path_translation/fallback/README.md b/acceptance/bundle/paths/fallback/README.md
similarity index 100%
rename from bundle/tests/path_translation/fallback/README.md
rename to acceptance/bundle/paths/fallback/README.md
diff --git a/bundle/tests/path_translation/nominal/databricks.yml b/acceptance/bundle/paths/fallback/databricks.yml
similarity index 80%
rename from bundle/tests/path_translation/nominal/databricks.yml
rename to acceptance/bundle/paths/fallback/databricks.yml
index cd425920d..c6d0abe0a 100644
--- a/bundle/tests/path_translation/nominal/databricks.yml
+++ b/acceptance/bundle/paths/fallback/databricks.yml
@@ -1,5 +1,5 @@
 bundle:
-  name: path_translation_nominal
+  name: fallback
 
 include:
   - "resources/*.yml"
diff --git a/acceptance/bundle/paths/fallback/output.job.json b/acceptance/bundle/paths/fallback/output.job.json
new file mode 100644
index 000000000..fe9e1cf3d
--- /dev/null
+++ b/acceptance/bundle/paths/fallback/output.job.json
@@ -0,0 +1,67 @@
+[
+  {
+    "job_cluster_key": "default",
+    "notebook_task": {
+      "notebook_path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/notebook"
+    },
+    "task_key": "notebook_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "spark_python_task": {
+      "python_file": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/file.py"
+    },
+    "task_key": "spark_python_example"
+  },
+  {
+    "dbt_task": {
+      "commands": [
+        "dbt run",
+        "dbt run"
+      ],
+      "project_directory": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/dbt_project"
+    },
+    "job_cluster_key": "default",
+    "task_key": "dbt_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "sql_task": {
+      "file": {
+        "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/sql.sql"
+      },
+      "warehouse_id": "cafef00d"
+    },
+    "task_key": "sql_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "libraries": [
+      {
+        "whl": "dist/wheel1.whl"
+      },
+      {
+        "whl": "dist/wheel2.whl"
+      }
+    ],
+    "python_wheel_task": {
+      "package_name": "my_package"
+    },
+    "task_key": "python_wheel_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "libraries": [
+      {
+        "jar": "target/jar1.jar"
+      },
+      {
+        "jar": "target/jar2.jar"
+      }
+    ],
+    "spark_jar_task": {
+      "main_class_name": "com.example.Main"
+    },
+    "task_key": "spark_jar_example"
+  }
+]
diff --git a/acceptance/bundle/paths/fallback/output.pipeline.json b/acceptance/bundle/paths/fallback/output.pipeline.json
new file mode 100644
index 000000000..38521cb22
--- /dev/null
+++ b/acceptance/bundle/paths/fallback/output.pipeline.json
@@ -0,0 +1,22 @@
+[
+  {
+    "file": {
+      "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/file1.py"
+    }
+  },
+  {
+    "notebook": {
+      "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/notebook1"
+    }
+  },
+  {
+    "file": {
+      "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/file2.py"
+    }
+  },
+  {
+    "notebook": {
+      "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/notebook2"
+    }
+  }
+]
diff --git a/acceptance/bundle/paths/fallback/output.txt b/acceptance/bundle/paths/fallback/output.txt
new file mode 100644
index 000000000..f694610d2
--- /dev/null
+++ b/acceptance/bundle/paths/fallback/output.txt
@@ -0,0 +1,18 @@
+
+>>> $CLI bundle validate -t development -o json
+
+Exit code: 0
+
+>>> $CLI bundle validate -t error
+Error: notebook this value is overridden not found. Local notebook references are expected
+to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb]
+
+Name: fallback
+Target: error
+Workspace:
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/fallback/error
+
+Found 1 error
+
+Exit code: 1
diff --git a/bundle/tests/path_translation/fallback/override_job.yml b/acceptance/bundle/paths/fallback/override_job.yml
similarity index 100%
rename from bundle/tests/path_translation/fallback/override_job.yml
rename to acceptance/bundle/paths/fallback/override_job.yml
diff --git a/bundle/tests/path_translation/fallback/override_pipeline.yml b/acceptance/bundle/paths/fallback/override_pipeline.yml
similarity index 100%
rename from bundle/tests/path_translation/fallback/override_pipeline.yml
rename to acceptance/bundle/paths/fallback/override_pipeline.yml
diff --git a/bundle/tests/path_translation/fallback/resources/my_job.yml b/acceptance/bundle/paths/fallback/resources/my_job.yml
similarity index 71%
rename from bundle/tests/path_translation/fallback/resources/my_job.yml
rename to acceptance/bundle/paths/fallback/resources/my_job.yml
index 4907df4f0..921ee412b 100644
--- a/bundle/tests/path_translation/fallback/resources/my_job.yml
+++ b/acceptance/bundle/paths/fallback/resources/my_job.yml
@@ -4,33 +4,45 @@ resources:
       name: "placeholder"
       tasks:
         - task_key: notebook_example
+          job_cluster_key: default
           notebook_task:
             notebook_path: "this value is overridden"
 
         - task_key: spark_python_example
+          job_cluster_key: default
           spark_python_task:
             python_file: "this value is overridden"
 
         - task_key: dbt_example
+          job_cluster_key: default
           dbt_task:
             project_directory: "this value is overridden"
             commands:
               - "dbt run"
 
         - task_key: sql_example
+          job_cluster_key: default
           sql_task:
             file:
               path: "this value is overridden"
             warehouse_id: cafef00d
 
         - task_key: python_wheel_example
+          job_cluster_key: default
           python_wheel_task:
             package_name: my_package
           libraries:
             - whl: ../dist/wheel1.whl
 
         - task_key: spark_jar_example
+          job_cluster_key: default
           spark_jar_task:
             main_class_name: com.example.Main
           libraries:
             - jar: ../target/jar1.jar
+
+      # Include a job cluster for completeness
+      job_clusters:
+        - job_cluster_key: default
+          new_cluster:
+            spark_version: 15.4.x-scala2.12
diff --git a/bundle/tests/path_translation/fallback/resources/my_pipeline.yml b/acceptance/bundle/paths/fallback/resources/my_pipeline.yml
similarity index 100%
rename from bundle/tests/path_translation/fallback/resources/my_pipeline.yml
rename to acceptance/bundle/paths/fallback/resources/my_pipeline.yml
diff --git a/acceptance/bundle/paths/fallback/script b/acceptance/bundle/paths/fallback/script
new file mode 100644
index 000000000..29aa420c5
--- /dev/null
+++ b/acceptance/bundle/paths/fallback/script
@@ -0,0 +1,10 @@
+errcode trace $CLI bundle validate -t development -o json > output.tmp.json
+
+# Capture job tasks
+jq '.resources.jobs.my_job.tasks' output.tmp.json > output.job.json
+
+# Capture pipeline libraries
+jq '.resources.pipelines.my_pipeline.libraries' output.tmp.json > output.pipeline.json
+
+# Expect failure for the "error" target
+errcode trace $CLI bundle validate -t error
diff --git a/acceptance/bundle/paths/fallback/script.cleanup b/acceptance/bundle/paths/fallback/script.cleanup
new file mode 100644
index 000000000..f93425dff
--- /dev/null
+++ b/acceptance/bundle/paths/fallback/script.cleanup
@@ -0,0 +1 @@
+rm -f output.tmp.json
diff --git a/bundle/tests/path_translation/fallback/src/dbt_project/.gitkeep b/acceptance/bundle/paths/fallback/src/dbt_project/.gitkeep
similarity index 100%
rename from bundle/tests/path_translation/fallback/src/dbt_project/.gitkeep
rename to acceptance/bundle/paths/fallback/src/dbt_project/.gitkeep
diff --git a/bundle/tests/path_translation/fallback/src/file.py b/acceptance/bundle/paths/fallback/src/file.py
similarity index 100%
rename from bundle/tests/path_translation/fallback/src/file.py
rename to acceptance/bundle/paths/fallback/src/file.py
diff --git a/bundle/tests/path_translation/fallback/src/file1.py b/acceptance/bundle/paths/fallback/src/file1.py
similarity index 100%
rename from bundle/tests/path_translation/fallback/src/file1.py
rename to acceptance/bundle/paths/fallback/src/file1.py
diff --git a/bundle/tests/path_translation/fallback/src/file2.py b/acceptance/bundle/paths/fallback/src/file2.py
similarity index 100%
rename from bundle/tests/path_translation/fallback/src/file2.py
rename to acceptance/bundle/paths/fallback/src/file2.py
diff --git a/bundle/tests/path_translation/fallback/src/notebook.py b/acceptance/bundle/paths/fallback/src/notebook.py
similarity index 100%
rename from bundle/tests/path_translation/fallback/src/notebook.py
rename to acceptance/bundle/paths/fallback/src/notebook.py
diff --git a/bundle/tests/path_translation/fallback/src/notebook1.py b/acceptance/bundle/paths/fallback/src/notebook1.py
similarity index 100%
rename from bundle/tests/path_translation/fallback/src/notebook1.py
rename to acceptance/bundle/paths/fallback/src/notebook1.py
diff --git a/bundle/tests/path_translation/fallback/src/notebook2.py b/acceptance/bundle/paths/fallback/src/notebook2.py
similarity index 100%
rename from bundle/tests/path_translation/fallback/src/notebook2.py
rename to acceptance/bundle/paths/fallback/src/notebook2.py
diff --git a/bundle/tests/path_translation/fallback/src/sql.sql b/acceptance/bundle/paths/fallback/src/sql.sql
similarity index 100%
rename from bundle/tests/path_translation/fallback/src/sql.sql
rename to acceptance/bundle/paths/fallback/src/sql.sql
diff --git a/bundle/tests/path_translation/nominal/README.md b/acceptance/bundle/paths/nominal/README.md
similarity index 100%
rename from bundle/tests/path_translation/nominal/README.md
rename to acceptance/bundle/paths/nominal/README.md
diff --git a/bundle/tests/path_translation/fallback/databricks.yml b/acceptance/bundle/paths/nominal/databricks.yml
similarity index 79%
rename from bundle/tests/path_translation/fallback/databricks.yml
rename to acceptance/bundle/paths/nominal/databricks.yml
index 92be3f921..5d3c22f91 100644
--- a/bundle/tests/path_translation/fallback/databricks.yml
+++ b/acceptance/bundle/paths/nominal/databricks.yml
@@ -1,5 +1,5 @@
 bundle:
-  name: path_translation_fallback
+  name: nominal
 
 include:
   - "resources/*.yml"
diff --git a/acceptance/bundle/paths/nominal/output.job.json b/acceptance/bundle/paths/nominal/output.job.json
new file mode 100644
index 000000000..9e1cb4d90
--- /dev/null
+++ b/acceptance/bundle/paths/nominal/output.job.json
@@ -0,0 +1,89 @@
+[
+  {
+    "job_cluster_key": "default",
+    "notebook_task": {
+      "notebook_path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook"
+    },
+    "task_key": "notebook_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "spark_python_task": {
+      "python_file": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file.py"
+    },
+    "task_key": "spark_python_example"
+  },
+  {
+    "dbt_task": {
+      "commands": [
+        "dbt run",
+        "dbt run"
+      ],
+      "project_directory": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/dbt_project"
+    },
+    "job_cluster_key": "default",
+    "task_key": "dbt_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "sql_task": {
+      "file": {
+        "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/sql.sql"
+      },
+      "warehouse_id": "cafef00d"
+    },
+    "task_key": "sql_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "libraries": [
+      {
+        "whl": "dist/wheel1.whl"
+      },
+      {
+        "whl": "dist/wheel2.whl"
+      }
+    ],
+    "python_wheel_task": {
+      "package_name": "my_package"
+    },
+    "task_key": "python_wheel_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "libraries": [
+      {
+        "jar": "target/jar1.jar"
+      },
+      {
+        "jar": "target/jar2.jar"
+      }
+    ],
+    "spark_jar_task": {
+      "main_class_name": "com.example.Main"
+    },
+    "task_key": "spark_jar_example"
+  },
+  {
+    "for_each_task": {
+      "task": {
+        "notebook_task": {
+          "notebook_path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook"
+        }
+      }
+    },
+    "job_cluster_key": "default",
+    "task_key": "for_each_notebook_example"
+  },
+  {
+    "for_each_task": {
+      "task": {
+        "job_cluster_key": "default",
+        "spark_python_task": {
+          "python_file": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file.py"
+        }
+      }
+    },
+    "task_key": "for_each_spark_python_example"
+  }
+]
diff --git a/acceptance/bundle/paths/nominal/output.pipeline.json b/acceptance/bundle/paths/nominal/output.pipeline.json
new file mode 100644
index 000000000..277b0c4a1
--- /dev/null
+++ b/acceptance/bundle/paths/nominal/output.pipeline.json
@@ -0,0 +1,22 @@
+[
+  {
+    "file": {
+      "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file1.py"
+    }
+  },
+  {
+    "notebook": {
+      "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook1"
+    }
+  },
+  {
+    "file": {
+      "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file2.py"
+    }
+  },
+  {
+    "notebook": {
+      "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook2"
+    }
+  }
+]
diff --git a/acceptance/bundle/paths/nominal/output.txt b/acceptance/bundle/paths/nominal/output.txt
new file mode 100644
index 000000000..189170335
--- /dev/null
+++ b/acceptance/bundle/paths/nominal/output.txt
@@ -0,0 +1,18 @@
+
+>>> $CLI bundle validate -t development -o json
+
+Exit code: 0
+
+>>> $CLI bundle validate -t error
+Error: notebook this value is overridden not found. Local notebook references are expected
+to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb]
+
+Name: nominal
+Target: error
+Workspace:
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/nominal/error
+
+Found 1 error
+
+Exit code: 1
diff --git a/bundle/tests/path_translation/nominal/override_job.yml b/acceptance/bundle/paths/nominal/override_job.yml
similarity index 100%
rename from bundle/tests/path_translation/nominal/override_job.yml
rename to acceptance/bundle/paths/nominal/override_job.yml
diff --git a/bundle/tests/path_translation/nominal/override_pipeline.yml b/acceptance/bundle/paths/nominal/override_pipeline.yml
similarity index 100%
rename from bundle/tests/path_translation/nominal/override_pipeline.yml
rename to acceptance/bundle/paths/nominal/override_pipeline.yml
diff --git a/bundle/tests/path_translation/nominal/resources/my_job.yml b/acceptance/bundle/paths/nominal/resources/my_job.yml
similarity index 74%
rename from bundle/tests/path_translation/nominal/resources/my_job.yml
rename to acceptance/bundle/paths/nominal/resources/my_job.yml
index 2020c9dc8..13996a20c 100644
--- a/bundle/tests/path_translation/nominal/resources/my_job.yml
+++ b/acceptance/bundle/paths/nominal/resources/my_job.yml
@@ -4,38 +4,45 @@ resources:
       name: "placeholder"
       tasks:
         - task_key: notebook_example
+          job_cluster_key: default
           notebook_task:
             notebook_path: "this value is overridden"
 
         - task_key: spark_python_example
+          job_cluster_key: default
           spark_python_task:
             python_file: "this value is overridden"
 
         - task_key: dbt_example
+          job_cluster_key: default
           dbt_task:
             project_directory: "this value is overridden"
             commands:
               - "dbt run"
 
         - task_key: sql_example
+          job_cluster_key: default
           sql_task:
             file:
               path: "this value is overridden"
             warehouse_id: cafef00d
 
         - task_key: python_wheel_example
+          job_cluster_key: default
           python_wheel_task:
             package_name: my_package
           libraries:
             - whl: ../dist/wheel1.whl
 
         - task_key: spark_jar_example
+          job_cluster_key: default
           spark_jar_task:
             main_class_name: com.example.Main
           libraries:
             - jar: ../target/jar1.jar
 
         - task_key: for_each_notebook_example
+          job_cluster_key: default
           for_each_task:
             task:
               notebook_task:
@@ -44,5 +51,12 @@ resources:
         - task_key: for_each_spark_python_example
           for_each_task:
             task:
+              job_cluster_key: default
               spark_python_task:
                 python_file: "this value is overridden"
+
+      # Include a job cluster for completeness
+      job_clusters:
+        - job_cluster_key: default
+          new_cluster:
+            spark_version: 15.4.x-scala2.12
diff --git a/bundle/tests/path_translation/nominal/resources/my_pipeline.yml b/acceptance/bundle/paths/nominal/resources/my_pipeline.yml
similarity index 100%
rename from bundle/tests/path_translation/nominal/resources/my_pipeline.yml
rename to acceptance/bundle/paths/nominal/resources/my_pipeline.yml
diff --git a/acceptance/bundle/paths/nominal/script b/acceptance/bundle/paths/nominal/script
new file mode 100644
index 000000000..29aa420c5
--- /dev/null
+++ b/acceptance/bundle/paths/nominal/script
@@ -0,0 +1,10 @@
+errcode trace $CLI bundle validate -t development -o json > output.tmp.json
+
+# Capture job tasks
+jq '.resources.jobs.my_job.tasks' output.tmp.json > output.job.json
+
+# Capture pipeline libraries
+jq '.resources.pipelines.my_pipeline.libraries' output.tmp.json > output.pipeline.json
+
+# Expect failure for the "error" target
+errcode trace $CLI bundle validate -t error
diff --git a/acceptance/bundle/paths/nominal/script.cleanup b/acceptance/bundle/paths/nominal/script.cleanup
new file mode 100644
index 000000000..f93425dff
--- /dev/null
+++ b/acceptance/bundle/paths/nominal/script.cleanup
@@ -0,0 +1 @@
+rm -f output.tmp.json
diff --git a/bundle/tests/path_translation/nominal/src/dbt_project/.gitkeep b/acceptance/bundle/paths/nominal/src/dbt_project/.gitkeep
similarity index 100%
rename from bundle/tests/path_translation/nominal/src/dbt_project/.gitkeep
rename to acceptance/bundle/paths/nominal/src/dbt_project/.gitkeep
diff --git a/bundle/tests/path_translation/nominal/src/file.py b/acceptance/bundle/paths/nominal/src/file.py
similarity index 100%
rename from bundle/tests/path_translation/nominal/src/file.py
rename to acceptance/bundle/paths/nominal/src/file.py
diff --git a/bundle/tests/path_translation/nominal/src/file1.py b/acceptance/bundle/paths/nominal/src/file1.py
similarity index 100%
rename from bundle/tests/path_translation/nominal/src/file1.py
rename to acceptance/bundle/paths/nominal/src/file1.py
diff --git a/bundle/tests/path_translation/nominal/src/file2.py b/acceptance/bundle/paths/nominal/src/file2.py
similarity index 100%
rename from bundle/tests/path_translation/nominal/src/file2.py
rename to acceptance/bundle/paths/nominal/src/file2.py
diff --git a/bundle/tests/path_translation/nominal/src/notebook.py b/acceptance/bundle/paths/nominal/src/notebook.py
similarity index 100%
rename from bundle/tests/path_translation/nominal/src/notebook.py
rename to acceptance/bundle/paths/nominal/src/notebook.py
diff --git a/bundle/tests/path_translation/nominal/src/notebook1.py b/acceptance/bundle/paths/nominal/src/notebook1.py
similarity index 100%
rename from bundle/tests/path_translation/nominal/src/notebook1.py
rename to acceptance/bundle/paths/nominal/src/notebook1.py
diff --git a/bundle/tests/path_translation/nominal/src/notebook2.py b/acceptance/bundle/paths/nominal/src/notebook2.py
similarity index 100%
rename from bundle/tests/path_translation/nominal/src/notebook2.py
rename to acceptance/bundle/paths/nominal/src/notebook2.py
diff --git a/bundle/tests/path_translation/nominal/src/sql.sql b/acceptance/bundle/paths/nominal/src/sql.sql
similarity index 100%
rename from bundle/tests/path_translation/nominal/src/sql.sql
rename to acceptance/bundle/paths/nominal/src/sql.sql
diff --git a/bundle/tests/relative_path_translation/databricks.yml b/acceptance/bundle/paths/relative_path_translation/databricks.yml
similarity index 100%
rename from bundle/tests/relative_path_translation/databricks.yml
rename to acceptance/bundle/paths/relative_path_translation/databricks.yml
diff --git a/acceptance/bundle/paths/relative_path_translation/output.default.json b/acceptance/bundle/paths/relative_path_translation/output.default.json
new file mode 100644
index 000000000..e2514b392
--- /dev/null
+++ b/acceptance/bundle/paths/relative_path_translation/output.default.json
@@ -0,0 +1,6 @@
+{
+  "paths": [
+    "/Workspace/remote/src/file1.py",
+    "/Workspace/remote/src/file1.py"
+  ]
+}
diff --git a/acceptance/bundle/paths/relative_path_translation/output.override.json b/acceptance/bundle/paths/relative_path_translation/output.override.json
new file mode 100644
index 000000000..729d2eaa0
--- /dev/null
+++ b/acceptance/bundle/paths/relative_path_translation/output.override.json
@@ -0,0 +1,6 @@
+{
+  "paths": [
+    "/Workspace/remote/src/file2.py",
+    "/Workspace/remote/src/file2.py"
+  ]
+}
diff --git a/acceptance/bundle/paths/relative_path_translation/output.txt b/acceptance/bundle/paths/relative_path_translation/output.txt
new file mode 100644
index 000000000..362f2ec7b
--- /dev/null
+++ b/acceptance/bundle/paths/relative_path_translation/output.txt
@@ -0,0 +1,4 @@
+
+>>> $CLI bundle validate -t default -o json
+
+>>> $CLI bundle validate -t override -o json
diff --git a/bundle/tests/relative_path_translation/resources/job.yml b/acceptance/bundle/paths/relative_path_translation/resources/job.yml
similarity index 66%
rename from bundle/tests/relative_path_translation/resources/job.yml
rename to acceptance/bundle/paths/relative_path_translation/resources/job.yml
index 93f121f25..9540ff1ad 100644
--- a/bundle/tests/relative_path_translation/resources/job.yml
+++ b/acceptance/bundle/paths/relative_path_translation/resources/job.yml
@@ -3,12 +3,20 @@ resources:
     job:
       tasks:
         - task_key: local
+          job_cluster_key: default
           spark_python_task:
             python_file: ../src/file1.py
 
         - task_key: variable_reference
+          job_cluster_key: default
           spark_python_task:
             # Note: this is a pure variable reference yet needs to persist the location
             # of the reference, not the location of the variable value.
             # Also see https://github.com/databricks/cli/issues/1330.
             python_file: ${var.file_path}
+
+      # Include a job cluster for completeness
+      job_clusters:
+        - job_cluster_key: default
+          new_cluster:
+            spark_version: 15.4.x-scala2.12
diff --git a/acceptance/bundle/paths/relative_path_translation/script b/acceptance/bundle/paths/relative_path_translation/script
new file mode 100644
index 000000000..252e9a07f
--- /dev/null
+++ b/acceptance/bundle/paths/relative_path_translation/script
@@ -0,0 +1,4 @@
+trace $CLI bundle validate -t default -o json | \
+    jq '{ paths: [.resources.jobs.job.tasks[].spark_python_task.python_file] }' > output.default.json
+trace $CLI bundle validate -t override -o json | \
+    jq '{ paths: [.resources.jobs.job.tasks[].spark_python_task.python_file] }' > output.override.json
diff --git a/bundle/tests/relative_path_translation/src/file1.py b/acceptance/bundle/paths/relative_path_translation/src/file1.py
similarity index 100%
rename from bundle/tests/relative_path_translation/src/file1.py
rename to acceptance/bundle/paths/relative_path_translation/src/file1.py
diff --git a/bundle/tests/relative_path_translation/src/file2.py b/acceptance/bundle/paths/relative_path_translation/src/file2.py
similarity index 100%
rename from bundle/tests/relative_path_translation/src/file2.py
rename to acceptance/bundle/paths/relative_path_translation/src/file2.py
diff --git a/bundle/tests/path_translation_test.go b/bundle/tests/path_translation_test.go
deleted file mode 100644
index 05702d2a2..000000000
--- a/bundle/tests/path_translation_test.go
+++ /dev/null
@@ -1,112 +0,0 @@
-package config_tests
-
-import (
-	"context"
-	"path/filepath"
-	"testing"
-
-	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/bundle/config/mutator"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestPathTranslationFallback(t *testing.T) {
-	b := loadTarget(t, "./path_translation/fallback", "development")
-
-	m := mutator.TranslatePaths()
-	diags := bundle.Apply(context.Background(), b, m)
-	require.NoError(t, diags.Error())
-
-	j := b.Config.Resources.Jobs["my_job"]
-	assert.Len(t, j.Tasks, 6)
-
-	assert.Equal(t, "notebook_example", filepath.ToSlash(j.Tasks[0].TaskKey))
-	assert.Equal(t, "src/notebook", filepath.ToSlash(j.Tasks[0].NotebookTask.NotebookPath))
-
-	assert.Equal(t, "spark_python_example", filepath.ToSlash(j.Tasks[1].TaskKey))
-	assert.Equal(t, "src/file.py", filepath.ToSlash(j.Tasks[1].SparkPythonTask.PythonFile))
-
-	assert.Equal(t, "dbt_example", filepath.ToSlash(j.Tasks[2].TaskKey))
-	assert.Equal(t, "src/dbt_project", filepath.ToSlash(j.Tasks[2].DbtTask.ProjectDirectory))
-
-	assert.Equal(t, "sql_example", filepath.ToSlash(j.Tasks[3].TaskKey))
-	assert.Equal(t, "src/sql.sql", filepath.ToSlash(j.Tasks[3].SqlTask.File.Path))
-
-	assert.Equal(t, "python_wheel_example", filepath.ToSlash(j.Tasks[4].TaskKey))
-	assert.Equal(t, "dist/wheel1.whl", filepath.ToSlash(j.Tasks[4].Libraries[0].Whl))
-	assert.Equal(t, "dist/wheel2.whl", filepath.ToSlash(j.Tasks[4].Libraries[1].Whl))
-
-	assert.Equal(t, "spark_jar_example", filepath.ToSlash(j.Tasks[5].TaskKey))
-	assert.Equal(t, "target/jar1.jar", filepath.ToSlash(j.Tasks[5].Libraries[0].Jar))
-	assert.Equal(t, "target/jar2.jar", filepath.ToSlash(j.Tasks[5].Libraries[1].Jar))
-
-	p := b.Config.Resources.Pipelines["my_pipeline"]
-	assert.Len(t, p.Libraries, 4)
-
-	assert.Equal(t, "src/file1.py", filepath.ToSlash(p.Libraries[0].File.Path))
-	assert.Equal(t, "src/notebook1", filepath.ToSlash(p.Libraries[1].Notebook.Path))
-	assert.Equal(t, "src/file2.py", filepath.ToSlash(p.Libraries[2].File.Path))
-	assert.Equal(t, "src/notebook2", filepath.ToSlash(p.Libraries[3].Notebook.Path))
-}
-
-func TestPathTranslationFallbackError(t *testing.T) {
-	b := loadTarget(t, "./path_translation/fallback", "error")
-
-	m := mutator.TranslatePaths()
-	diags := bundle.Apply(context.Background(), b, m)
-	assert.ErrorContains(t, diags.Error(), `notebook this value is overridden not found`)
-}
-
-func TestPathTranslationNominal(t *testing.T) {
-	b := loadTarget(t, "./path_translation/nominal", "development")
-
-	m := mutator.TranslatePaths()
-	diags := bundle.Apply(context.Background(), b, m)
-	assert.NoError(t, diags.Error())
-
-	j := b.Config.Resources.Jobs["my_job"]
-	assert.Len(t, j.Tasks, 8)
-
-	assert.Equal(t, "notebook_example", filepath.ToSlash(j.Tasks[0].TaskKey))
-	assert.Equal(t, "src/notebook", filepath.ToSlash(j.Tasks[0].NotebookTask.NotebookPath))
-
-	assert.Equal(t, "spark_python_example", filepath.ToSlash(j.Tasks[1].TaskKey))
-	assert.Equal(t, "src/file.py", filepath.ToSlash(j.Tasks[1].SparkPythonTask.PythonFile))
-
-	assert.Equal(t, "dbt_example", filepath.ToSlash(j.Tasks[2].TaskKey))
-	assert.Equal(t, "src/dbt_project", filepath.ToSlash(j.Tasks[2].DbtTask.ProjectDirectory))
-
-	assert.Equal(t, "sql_example", filepath.ToSlash(j.Tasks[3].TaskKey))
-	assert.Equal(t, "src/sql.sql", filepath.ToSlash(j.Tasks[3].SqlTask.File.Path))
-
-	assert.Equal(t, "python_wheel_example", filepath.ToSlash(j.Tasks[4].TaskKey))
-	assert.Equal(t, "dist/wheel1.whl", filepath.ToSlash(j.Tasks[4].Libraries[0].Whl))
-	assert.Equal(t, "dist/wheel2.whl", filepath.ToSlash(j.Tasks[4].Libraries[1].Whl))
-
-	assert.Equal(t, "spark_jar_example", filepath.ToSlash(j.Tasks[5].TaskKey))
-	assert.Equal(t, "target/jar1.jar", filepath.ToSlash(j.Tasks[5].Libraries[0].Jar))
-	assert.Equal(t, "target/jar2.jar", filepath.ToSlash(j.Tasks[5].Libraries[1].Jar))
-
-	assert.Equal(t, "for_each_notebook_example", filepath.ToSlash(j.Tasks[6].TaskKey))
-	assert.Equal(t, "src/notebook", filepath.ToSlash(j.Tasks[6].ForEachTask.Task.NotebookTask.NotebookPath))
-
-	assert.Equal(t, "for_each_spark_python_example", filepath.ToSlash(j.Tasks[7].TaskKey))
-	assert.Equal(t, "src/file.py", filepath.ToSlash(j.Tasks[7].ForEachTask.Task.SparkPythonTask.PythonFile))
-
-	p := b.Config.Resources.Pipelines["my_pipeline"]
-	assert.Len(t, p.Libraries, 4)
-
-	assert.Equal(t, "src/file1.py", filepath.ToSlash(p.Libraries[0].File.Path))
-	assert.Equal(t, "src/notebook1", filepath.ToSlash(p.Libraries[1].Notebook.Path))
-	assert.Equal(t, "src/file2.py", filepath.ToSlash(p.Libraries[2].File.Path))
-	assert.Equal(t, "src/notebook2", filepath.ToSlash(p.Libraries[3].Notebook.Path))
-}
-
-func TestPathTranslationNominalError(t *testing.T) {
-	b := loadTarget(t, "./path_translation/nominal", "error")
-
-	m := mutator.TranslatePaths()
-	diags := bundle.Apply(context.Background(), b, m)
-	assert.ErrorContains(t, diags.Error(), `notebook this value is overridden not found`)
-}
diff --git a/bundle/tests/relative_path_translation_test.go b/bundle/tests/relative_path_translation_test.go
deleted file mode 100644
index 0f553ac3d..000000000
--- a/bundle/tests/relative_path_translation_test.go
+++ /dev/null
@@ -1,28 +0,0 @@
-package config_tests
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestRelativePathTranslationDefault(t *testing.T) {
-	b, diags := initializeTarget(t, "./relative_path_translation", "default")
-	require.NoError(t, diags.Error())
-
-	t0 := b.Config.Resources.Jobs["job"].Tasks[0]
-	assert.Equal(t, "/Workspace/remote/src/file1.py", t0.SparkPythonTask.PythonFile)
-	t1 := b.Config.Resources.Jobs["job"].Tasks[1]
-	assert.Equal(t, "/Workspace/remote/src/file1.py", t1.SparkPythonTask.PythonFile)
-}
-
-func TestRelativePathTranslationOverride(t *testing.T) {
-	b, diags := initializeTarget(t, "./relative_path_translation", "override")
-	require.NoError(t, diags.Error())
-
-	t0 := b.Config.Resources.Jobs["job"].Tasks[0]
-	assert.Equal(t, "/Workspace/remote/src/file2.py", t0.SparkPythonTask.PythonFile)
-	t1 := b.Config.Resources.Jobs["job"].Tasks[1]
-	assert.Equal(t, "/Workspace/remote/src/file2.py", t1.SparkPythonTask.PythonFile)
-}

From 560c3d352e886aef6d3a2f1a38cb91e4f9da83f2 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 17 Jan 2025 14:40:19 +0100
Subject: [PATCH 102/247] Add test for passing --var twice for the same arg
 (#2176)

This shows that passing two --var for the same arg is rejected
currently.
---
 .../variables/arg-repeat/databricks.yml       |  6 ++++++
 .../bundle/variables/arg-repeat/output.txt    | 20 +++++++++++++++++++
 acceptance/bundle/variables/arg-repeat/script |  2 ++
 3 files changed, 28 insertions(+)
 create mode 100644 acceptance/bundle/variables/arg-repeat/databricks.yml
 create mode 100644 acceptance/bundle/variables/arg-repeat/output.txt
 create mode 100644 acceptance/bundle/variables/arg-repeat/script

diff --git a/acceptance/bundle/variables/arg-repeat/databricks.yml b/acceptance/bundle/variables/arg-repeat/databricks.yml
new file mode 100644
index 000000000..377c6cfab
--- /dev/null
+++ b/acceptance/bundle/variables/arg-repeat/databricks.yml
@@ -0,0 +1,6 @@
+bundle:
+  name: arg-repeat
+
+variables:
+  a:
+    default: hello
diff --git a/acceptance/bundle/variables/arg-repeat/output.txt b/acceptance/bundle/variables/arg-repeat/output.txt
new file mode 100644
index 000000000..48bd2033f
--- /dev/null
+++ b/acceptance/bundle/variables/arg-repeat/output.txt
@@ -0,0 +1,20 @@
+
+>>> errcode $CLI bundle validate --var a=one -o json
+
+Exit code: 0
+{
+  "a": {
+    "default": "hello",
+    "value": "one"
+  }
+}
+
+>>> errcode $CLI bundle validate --var a=one --var a=two
+Error: failed to assign two to a: variable has already been assigned value: one
+
+Name: arg-repeat
+Target: default
+
+Found 1 error
+
+Exit code: 1
diff --git a/acceptance/bundle/variables/arg-repeat/script b/acceptance/bundle/variables/arg-repeat/script
new file mode 100644
index 000000000..3e03dbcb1
--- /dev/null
+++ b/acceptance/bundle/variables/arg-repeat/script
@@ -0,0 +1,2 @@
+trace errcode $CLI bundle validate --var a=one -o json | jq .variables
+trace errcode $CLI bundle validate --var a=one --var a=two

From 0c088d40501f724486cb82baf5aab2634668fc84 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Fri, 17 Jan 2025 15:42:39 +0100
Subject: [PATCH 103/247] Fixed an apps message order and added output test
 (#2174)

## Changes
Fixed an apps message order and added output test
---
 bundle/phases/deploy.go                       |  2 +-
 integration/bundle/apps_test.go               | 28 ++++++++++++++++++-
 .../bundle/testdata/apps/bundle_deploy.txt    |  5 ++++
 .../bundle/testdata/apps/bundle_validate.txt  |  7 +++++
 4 files changed, 40 insertions(+), 2 deletions(-)
 create mode 100644 integration/bundle/testdata/apps/bundle_deploy.txt
 create mode 100644 integration/bundle/testdata/apps/bundle_validate.txt

diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go
index 1a1ccd47b..b59ce9f89 100644
--- a/bundle/phases/deploy.go
+++ b/bundle/phases/deploy.go
@@ -130,13 +130,13 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator {
 	// mutators need informed consent if they are potentially destructive.
 	deployCore := bundle.Defer(
 		bundle.Seq(
+			apps.SlowDeployMessage(),
 			bundle.LogString("Deploying resources..."),
 			terraform.Apply(),
 		),
 		bundle.Seq(
 			terraform.StatePush(),
 			terraform.Load(),
-			apps.SlowDeployMessage(),
 			apps.InterpolateVariables(),
 			apps.UploadConfig(),
 			metadata.Compute(),
diff --git a/integration/bundle/apps_test.go b/integration/bundle/apps_test.go
index 2da764ed8..23cd784be 100644
--- a/integration/bundle/apps_test.go
+++ b/integration/bundle/apps_test.go
@@ -6,8 +6,10 @@ import (
 	"testing"
 
 	"github.com/databricks/cli/integration/internal/acc"
+	"github.com/databricks/cli/internal/testcli"
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/env"
+	"github.com/databricks/cli/libs/testdiff"
 	"github.com/databricks/databricks-sdk-go/service/apps"
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
@@ -49,7 +51,31 @@ func TestDeployBundleWithApp(t *testing.T) {
 		}
 	})
 
-	deployBundle(t, ctx, root)
+	ctx, replacements := testdiff.WithReplacementsMap(ctx)
+	replacements.Set(uniqueId, "$UNIQUE_PRJ")
+
+	user, err := wt.W.CurrentUser.Me(ctx)
+	require.NoError(t, err)
+	require.NotNil(t, user)
+	testdiff.PrepareReplacementsUser(t, replacements, *user)
+	testdiff.PrepareReplacementsWorkspaceClient(t, replacements, wt.W)
+	testdiff.PrepareReplacementsUUID(t, replacements)
+	testdiff.PrepareReplacementsNumber(t, replacements)
+	testdiff.PrepareReplacementsTemporaryDirectory(t, replacements)
+
+	testutil.Chdir(t, root)
+	testcli.AssertOutput(
+		t,
+		ctx,
+		[]string{"bundle", "validate"},
+		testutil.TestData("testdata/apps/bundle_validate.txt"),
+	)
+	testcli.AssertOutput(
+		t,
+		ctx,
+		[]string{"bundle", "deploy", "--force-lock", "--auto-approve"},
+		testutil.TestData("testdata/apps/bundle_deploy.txt"),
+	)
 
 	// App should exists after bundle deployment
 	app, err := wt.W.Apps.Get(ctx, apps.GetAppRequest{Name: appId})
diff --git a/integration/bundle/testdata/apps/bundle_deploy.txt b/integration/bundle/testdata/apps/bundle_deploy.txt
new file mode 100644
index 000000000..b077f327d
--- /dev/null
+++ b/integration/bundle/testdata/apps/bundle_deploy.txt
@@ -0,0 +1,5 @@
+Uploading bundle files to /Workspace/Users/$USERNAME/.bundle/$UNIQUE_PRJ/files...
+Note: Databricks apps included in this bundle may increase initial deployment time due to compute provisioning.
+Deploying resources...
+Updating deployment state...
+Deployment complete!
diff --git a/integration/bundle/testdata/apps/bundle_validate.txt b/integration/bundle/testdata/apps/bundle_validate.txt
new file mode 100644
index 000000000..dc9016a0f
--- /dev/null
+++ b/integration/bundle/testdata/apps/bundle_validate.txt
@@ -0,0 +1,7 @@
+Name: basic
+Target: default
+Workspace:
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/$UNIQUE_PRJ
+
+Validation OK!

From cff4f09cc80bc858c2894446ffa01b5448d3ebb0 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Fri, 17 Jan 2025 15:42:44 +0100
Subject: [PATCH 104/247] Added text output templates for apps list and
 list-deployments (#2175)

## Changes
Added text output templates for apps list and list-deployments

Fixes #2172

## Tests
```
andrew.nester@HFW9Y94129 ~ % databricks apps list -p u2m
Name                          Url                                                            Compute Status  Deployment Status
abc                           https://abc-***.aws.databricksapps.com                         STOPPED
andre-test                    https://andre-test-***..aws.databricksapps.com                  ACTIVE          SUCCEEDED
andre-test2                   https://andre-test2-***..aws.databricksapps.com                 ACTIVE          SUCCEEDED
...
```
---
 cmd/workspace/apps/overrides.go | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 cmd/workspace/apps/overrides.go

diff --git a/cmd/workspace/apps/overrides.go b/cmd/workspace/apps/overrides.go
new file mode 100644
index 000000000..e14068717
--- /dev/null
+++ b/cmd/workspace/apps/overrides.go
@@ -0,0 +1,28 @@
+package apps
+
+import (
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/databricks-sdk-go/service/apps"
+	"github.com/spf13/cobra"
+)
+
+func listOverride(listCmd *cobra.Command, listReq *apps.ListAppsRequest) {
+	listCmd.Annotations["headerTemplate"] = cmdio.Heredoc(`
+	{{header "Name"}}	{{header "Url"}}	{{header "ComputeStatus"}}	{{header "DeploymentStatus"}}`)
+	listCmd.Annotations["template"] = cmdio.Heredoc(`
+	{{range .}}{{.Name | green}}	{{.Url}}	{{if .ComputeStatus}}{{if eq .ComputeStatus.State "ACTIVE"}}{{green "%s" .ComputeStatus.State }}{{else}}{{blue "%s" .ComputeStatus.State}}{{end}}{{end}}	{{if .ActiveDeployment}}{{if eq .ActiveDeployment.Status.State "SUCCEEDED"}}{{green "%s" .ActiveDeployment.Status.State }}{{else}}{{blue "%s" .ActiveDeployment.Status.State}}{{end}}{{end}}
+	{{end}}`)
+}
+
+func listDeploymentsOverride(listDeploymentsCmd *cobra.Command, listDeploymentsReq *apps.ListAppDeploymentsRequest) {
+	listDeploymentsCmd.Annotations["headerTemplate"] = cmdio.Heredoc(`
+	{{header "DeploymentId"}}	{{header "State"}}	{{header "CreatedAt"}}`)
+	listDeploymentsCmd.Annotations["template"] = cmdio.Heredoc(`
+	{{range .}}{{.DeploymentId}}	{{if eq .Status.State "SUCCEEDED"}}{{green "%s" .Status.State }}{{else}}{{blue "%s" .Status.State}}{{end}}	{{.CreateTime}}
+	{{end}}`)
+}
+
+func init() {
+	listOverrides = append(listOverrides, listOverride)
+	listDeploymentsOverrides = append(listDeploymentsOverrides, listDeploymentsOverride)
+}

From 0d5193a62c28b2ca1253d1753862d7900163e3f6 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Fri, 17 Jan 2025 15:52:53 +0100
Subject: [PATCH 105/247] Include help output for bundle commands in acceptance
 tests (#2178)

## Changes

This includes a change to the defaults for the output directory flags of
the "generate" commands. These defaults included the expanded working
directory. This can be omitted because it is implied.
---
 .../bundle/help/bundle-deploy/output.txt      | 21 +++++++
 acceptance/bundle/help/bundle-deploy/script   |  1 +
 .../bundle/help/bundle-deployment/output.txt  | 22 +++++++
 .../bundle/help/bundle-deployment/script      |  1 +
 .../bundle/help/bundle-destroy/output.txt     | 18 ++++++
 acceptance/bundle/help/bundle-destroy/script  |  1 +
 .../help/bundle-generate-dashboard/output.txt | 24 ++++++++
 .../help/bundle-generate-dashboard/script     |  1 +
 .../help/bundle-generate-job/output.txt       | 21 +++++++
 .../bundle/help/bundle-generate-job/script    |  1 +
 .../help/bundle-generate-pipeline/output.txt  | 21 +++++++
 .../help/bundle-generate-pipeline/script      |  1 +
 .../bundle/help/bundle-generate/output.txt    | 25 ++++++++
 acceptance/bundle/help/bundle-generate/script |  1 +
 acceptance/bundle/help/bundle-init/output.txt | 31 ++++++++++
 acceptance/bundle/help/bundle-init/script     |  1 +
 acceptance/bundle/help/bundle-open/output.txt | 17 ++++++
 acceptance/bundle/help/bundle-open/script     |  1 +
 acceptance/bundle/help/bundle-run/output.txt  | 57 +++++++++++++++++++
 acceptance/bundle/help/bundle-run/script      |  1 +
 .../bundle/help/bundle-schema/output.txt      | 16 ++++++
 acceptance/bundle/help/bundle-schema/script   |  1 +
 .../bundle/help/bundle-summary/output.txt     | 17 ++++++
 acceptance/bundle/help/bundle-summary/script  |  1 +
 acceptance/bundle/help/bundle-sync/output.txt | 19 +++++++
 acceptance/bundle/help/bundle-sync/script     |  1 +
 .../bundle/help/bundle-validate/output.txt    | 16 ++++++
 acceptance/bundle/help/bundle-validate/script |  1 +
 acceptance/bundle/help/bundle/output.txt      | 33 +++++++++++
 acceptance/bundle/help/bundle/script          |  1 +
 cmd/bundle/generate/app.go                    |  4 +-
 cmd/bundle/generate/dashboard.go              |  4 +-
 cmd/bundle/generate/job.go                    |  9 +--
 cmd/bundle/generate/pipeline.go               |  9 +--
 34 files changed, 381 insertions(+), 18 deletions(-)
 create mode 100644 acceptance/bundle/help/bundle-deploy/output.txt
 create mode 100644 acceptance/bundle/help/bundle-deploy/script
 create mode 100644 acceptance/bundle/help/bundle-deployment/output.txt
 create mode 100644 acceptance/bundle/help/bundle-deployment/script
 create mode 100644 acceptance/bundle/help/bundle-destroy/output.txt
 create mode 100644 acceptance/bundle/help/bundle-destroy/script
 create mode 100644 acceptance/bundle/help/bundle-generate-dashboard/output.txt
 create mode 100644 acceptance/bundle/help/bundle-generate-dashboard/script
 create mode 100644 acceptance/bundle/help/bundle-generate-job/output.txt
 create mode 100644 acceptance/bundle/help/bundle-generate-job/script
 create mode 100644 acceptance/bundle/help/bundle-generate-pipeline/output.txt
 create mode 100644 acceptance/bundle/help/bundle-generate-pipeline/script
 create mode 100644 acceptance/bundle/help/bundle-generate/output.txt
 create mode 100644 acceptance/bundle/help/bundle-generate/script
 create mode 100644 acceptance/bundle/help/bundle-init/output.txt
 create mode 100644 acceptance/bundle/help/bundle-init/script
 create mode 100644 acceptance/bundle/help/bundle-open/output.txt
 create mode 100644 acceptance/bundle/help/bundle-open/script
 create mode 100644 acceptance/bundle/help/bundle-run/output.txt
 create mode 100644 acceptance/bundle/help/bundle-run/script
 create mode 100644 acceptance/bundle/help/bundle-schema/output.txt
 create mode 100644 acceptance/bundle/help/bundle-schema/script
 create mode 100644 acceptance/bundle/help/bundle-summary/output.txt
 create mode 100644 acceptance/bundle/help/bundle-summary/script
 create mode 100644 acceptance/bundle/help/bundle-sync/output.txt
 create mode 100644 acceptance/bundle/help/bundle-sync/script
 create mode 100644 acceptance/bundle/help/bundle-validate/output.txt
 create mode 100644 acceptance/bundle/help/bundle-validate/script
 create mode 100644 acceptance/bundle/help/bundle/output.txt
 create mode 100644 acceptance/bundle/help/bundle/script

diff --git a/acceptance/bundle/help/bundle-deploy/output.txt b/acceptance/bundle/help/bundle-deploy/output.txt
new file mode 100644
index 000000000..13c903f3e
--- /dev/null
+++ b/acceptance/bundle/help/bundle-deploy/output.txt
@@ -0,0 +1,21 @@
+
+>>> $CLI bundle deploy --help
+Deploy bundle
+
+Usage:
+  databricks bundle deploy [flags]
+
+Flags:
+      --auto-approve          Skip interactive approvals that might be required for deployment.
+  -c, --cluster-id string     Override cluster in the deployment with the given cluster ID.
+      --fail-on-active-runs   Fail if there are running jobs or pipelines in the deployment.
+      --force                 Force-override Git branch validation.
+      --force-lock            Force acquisition of deployment lock.
+  -h, --help                  help for deploy
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+      --var strings      set values for variables defined in bundle config. Example: --var="foo=bar"
diff --git a/acceptance/bundle/help/bundle-deploy/script b/acceptance/bundle/help/bundle-deploy/script
new file mode 100644
index 000000000..6375cfea3
--- /dev/null
+++ b/acceptance/bundle/help/bundle-deploy/script
@@ -0,0 +1 @@
+trace $CLI bundle deploy --help
diff --git a/acceptance/bundle/help/bundle-deployment/output.txt b/acceptance/bundle/help/bundle-deployment/output.txt
new file mode 100644
index 000000000..ddf5b3305
--- /dev/null
+++ b/acceptance/bundle/help/bundle-deployment/output.txt
@@ -0,0 +1,22 @@
+
+>>> $CLI bundle deployment --help
+Deployment related commands
+
+Usage:
+  databricks bundle deployment [command]
+
+Available Commands:
+  bind        Bind bundle-defined resources to existing resources
+  unbind      Unbind bundle-defined resources from its managed remote resource
+
+Flags:
+  -h, --help   help for deployment
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+      --var strings      set values for variables defined in bundle config. Example: --var="foo=bar"
+
+Use "databricks bundle deployment [command] --help" for more information about a command.
diff --git a/acceptance/bundle/help/bundle-deployment/script b/acceptance/bundle/help/bundle-deployment/script
new file mode 100644
index 000000000..ef93f7dc2
--- /dev/null
+++ b/acceptance/bundle/help/bundle-deployment/script
@@ -0,0 +1 @@
+trace $CLI bundle deployment --help
diff --git a/acceptance/bundle/help/bundle-destroy/output.txt b/acceptance/bundle/help/bundle-destroy/output.txt
new file mode 100644
index 000000000..d70164301
--- /dev/null
+++ b/acceptance/bundle/help/bundle-destroy/output.txt
@@ -0,0 +1,18 @@
+
+>>> $CLI bundle destroy --help
+Destroy deployed bundle resources
+
+Usage:
+  databricks bundle destroy [flags]
+
+Flags:
+      --auto-approve   Skip interactive approvals for deleting resources and files
+      --force-lock     Force acquisition of deployment lock.
+  -h, --help           help for destroy
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+      --var strings      set values for variables defined in bundle config. Example: --var="foo=bar"
diff --git a/acceptance/bundle/help/bundle-destroy/script b/acceptance/bundle/help/bundle-destroy/script
new file mode 100644
index 000000000..955d7b7f9
--- /dev/null
+++ b/acceptance/bundle/help/bundle-destroy/script
@@ -0,0 +1 @@
+trace $CLI bundle destroy --help
diff --git a/acceptance/bundle/help/bundle-generate-dashboard/output.txt b/acceptance/bundle/help/bundle-generate-dashboard/output.txt
new file mode 100644
index 000000000..a63ce0ff8
--- /dev/null
+++ b/acceptance/bundle/help/bundle-generate-dashboard/output.txt
@@ -0,0 +1,24 @@
+
+>>> $CLI bundle generate dashboard --help
+Generate configuration for a dashboard
+
+Usage:
+  databricks bundle generate dashboard [flags]
+
+Flags:
+  -s, --dashboard-dir string   directory to write the dashboard representation to (default "src")
+      --existing-id string     ID of the dashboard to generate configuration for
+      --existing-path string   workspace path of the dashboard to generate configuration for
+  -f, --force                  force overwrite existing files in the output directory
+  -h, --help                   help for dashboard
+      --resource string        resource key of dashboard to watch for changes
+  -d, --resource-dir string    directory to write the configuration to (default "resources")
+      --watch                  watch for changes to the dashboard and update the configuration
+
+Global Flags:
+      --debug            enable debug logging
+      --key string       resource key to use for the generated configuration
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+      --var strings      set values for variables defined in bundle config. Example: --var="foo=bar"
diff --git a/acceptance/bundle/help/bundle-generate-dashboard/script b/acceptance/bundle/help/bundle-generate-dashboard/script
new file mode 100644
index 000000000..320156129
--- /dev/null
+++ b/acceptance/bundle/help/bundle-generate-dashboard/script
@@ -0,0 +1 @@
+trace $CLI bundle generate dashboard --help
diff --git a/acceptance/bundle/help/bundle-generate-job/output.txt b/acceptance/bundle/help/bundle-generate-job/output.txt
new file mode 100644
index 000000000..adc3f45ae
--- /dev/null
+++ b/acceptance/bundle/help/bundle-generate-job/output.txt
@@ -0,0 +1,21 @@
+
+>>> $CLI bundle generate job --help
+Generate bundle configuration for a job
+
+Usage:
+  databricks bundle generate job [flags]
+
+Flags:
+  -d, --config-dir string     Dir path where the output config will be stored (default "resources")
+      --existing-job-id int   Job ID of the job to generate config for
+  -f, --force                 Force overwrite existing files in the output directory
+  -h, --help                  help for job
+  -s, --source-dir string     Dir path where the downloaded files will be stored (default "src")
+
+Global Flags:
+      --debug            enable debug logging
+      --key string       resource key to use for the generated configuration
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+      --var strings      set values for variables defined in bundle config. Example: --var="foo=bar"
diff --git a/acceptance/bundle/help/bundle-generate-job/script b/acceptance/bundle/help/bundle-generate-job/script
new file mode 100644
index 000000000..109ed59aa
--- /dev/null
+++ b/acceptance/bundle/help/bundle-generate-job/script
@@ -0,0 +1 @@
+trace $CLI bundle generate job --help
diff --git a/acceptance/bundle/help/bundle-generate-pipeline/output.txt b/acceptance/bundle/help/bundle-generate-pipeline/output.txt
new file mode 100644
index 000000000..cf5f70920
--- /dev/null
+++ b/acceptance/bundle/help/bundle-generate-pipeline/output.txt
@@ -0,0 +1,21 @@
+
+>>> $CLI bundle generate pipeline --help
+Generate bundle configuration for a pipeline
+
+Usage:
+  databricks bundle generate pipeline [flags]
+
+Flags:
+  -d, --config-dir string             Dir path where the output config will be stored (default "resources")
+      --existing-pipeline-id string   ID of the pipeline to generate config for
+  -f, --force                         Force overwrite existing files in the output directory
+  -h, --help                          help for pipeline
+  -s, --source-dir string             Dir path where the downloaded files will be stored (default "src")
+
+Global Flags:
+      --debug            enable debug logging
+      --key string       resource key to use for the generated configuration
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+      --var strings      set values for variables defined in bundle config. Example: --var="foo=bar"
diff --git a/acceptance/bundle/help/bundle-generate-pipeline/script b/acceptance/bundle/help/bundle-generate-pipeline/script
new file mode 100644
index 000000000..c6af62d0a
--- /dev/null
+++ b/acceptance/bundle/help/bundle-generate-pipeline/script
@@ -0,0 +1 @@
+trace $CLI bundle generate pipeline --help
diff --git a/acceptance/bundle/help/bundle-generate/output.txt b/acceptance/bundle/help/bundle-generate/output.txt
new file mode 100644
index 000000000..1d77dfdbd
--- /dev/null
+++ b/acceptance/bundle/help/bundle-generate/output.txt
@@ -0,0 +1,25 @@
+
+>>> $CLI bundle generate --help
+Generate bundle configuration
+
+Usage:
+  databricks bundle generate [command]
+
+Available Commands:
+  app         Generate bundle configuration for a Databricks app
+  dashboard   Generate configuration for a dashboard
+  job         Generate bundle configuration for a job
+  pipeline    Generate bundle configuration for a pipeline
+
+Flags:
+  -h, --help         help for generate
+      --key string   resource key to use for the generated configuration
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+      --var strings      set values for variables defined in bundle config. Example: --var="foo=bar"
+
+Use "databricks bundle generate [command] --help" for more information about a command.
diff --git a/acceptance/bundle/help/bundle-generate/script b/acceptance/bundle/help/bundle-generate/script
new file mode 100644
index 000000000..932588768
--- /dev/null
+++ b/acceptance/bundle/help/bundle-generate/script
@@ -0,0 +1 @@
+trace $CLI bundle generate --help
diff --git a/acceptance/bundle/help/bundle-init/output.txt b/acceptance/bundle/help/bundle-init/output.txt
new file mode 100644
index 000000000..bafe5a187
--- /dev/null
+++ b/acceptance/bundle/help/bundle-init/output.txt
@@ -0,0 +1,31 @@
+
+>>> $CLI bundle init --help
+Initialize using a bundle template.
+
+TEMPLATE_PATH optionally specifies which template to use. It can be one of the following:
+- default-python: The default Python template for Notebooks / Delta Live Tables / Workflows
+- default-sql: The default SQL template for .sql files that run with Databricks SQL
+- dbt-sql: The dbt SQL template (databricks.com/blog/delivering-cost-effective-data-real-time-dbt-and-databricks)
+- mlops-stacks: The Databricks MLOps Stacks template (github.com/databricks/mlops-stacks)
+- a local file system path with a template directory
+- a Git repository URL, e.g. https://github.com/my/repository
+
+See https://docs.databricks.com/en/dev-tools/bundles/templates.html for more information on templates.
+
+Usage:
+  databricks bundle init [TEMPLATE_PATH] [flags]
+
+Flags:
+      --branch string         Git branch to use for template initialization
+      --config-file string    JSON file containing key value pairs of input parameters required for template initialization.
+  -h, --help                  help for init
+      --output-dir string     Directory to write the initialized template to.
+      --tag string            Git tag to use for template initialization
+      --template-dir string   Directory path within a Git repository containing the template.
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+      --var strings      set values for variables defined in bundle config. Example: --var="foo=bar"
diff --git a/acceptance/bundle/help/bundle-init/script b/acceptance/bundle/help/bundle-init/script
new file mode 100644
index 000000000..9bcce7559
--- /dev/null
+++ b/acceptance/bundle/help/bundle-init/script
@@ -0,0 +1 @@
+trace $CLI bundle init --help
diff --git a/acceptance/bundle/help/bundle-open/output.txt b/acceptance/bundle/help/bundle-open/output.txt
new file mode 100644
index 000000000..8b98aa850
--- /dev/null
+++ b/acceptance/bundle/help/bundle-open/output.txt
@@ -0,0 +1,17 @@
+
+>>> $CLI bundle open --help
+Open a resource in the browser
+
+Usage:
+  databricks bundle open [flags]
+
+Flags:
+      --force-pull   Skip local cache and load the state from the remote workspace
+  -h, --help         help for open
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+      --var strings      set values for variables defined in bundle config. Example: --var="foo=bar"
diff --git a/acceptance/bundle/help/bundle-open/script b/acceptance/bundle/help/bundle-open/script
new file mode 100644
index 000000000..b4dfa2222
--- /dev/null
+++ b/acceptance/bundle/help/bundle-open/script
@@ -0,0 +1 @@
+trace $CLI bundle open --help
diff --git a/acceptance/bundle/help/bundle-run/output.txt b/acceptance/bundle/help/bundle-run/output.txt
new file mode 100644
index 000000000..17763a295
--- /dev/null
+++ b/acceptance/bundle/help/bundle-run/output.txt
@@ -0,0 +1,57 @@
+
+>>> $CLI bundle run --help
+Run the job or pipeline identified by KEY.
+
+The KEY is the unique identifier of the resource to run. In addition to
+customizing the run using any of the available flags, you can also specify
+keyword or positional arguments as shown in these examples:
+
+   databricks bundle run my_job -- --key1 value1 --key2 value2
+
+Or:
+
+   databricks bundle run my_job -- value1 value2 value3
+
+If the specified job uses job parameters or the job has a notebook task with
+parameters, the first example applies and flag names are mapped to the
+parameter names.
+
+If the specified job does not use job parameters and the job has a Python file
+task or a Python wheel task, the second example applies.
+
+Usage:
+  databricks bundle run [flags] KEY
+
+Job Flags:
+      --params stringToString   comma separated k=v pairs for job parameters (default [])
+
+Job Task Flags:
+  Note: please prefer use of job-level parameters (--param) over task-level parameters.
+  For more information, see https://docs.databricks.com/en/workflows/jobs/create-run-jobs.html#pass-parameters-to-a-databricks-job-task
+      --dbt-commands strings                 A list of commands to execute for jobs with DBT tasks.
+      --jar-params strings                   A list of parameters for jobs with Spark JAR tasks.
+      --notebook-params stringToString       A map from keys to values for jobs with notebook tasks. (default [])
+      --pipeline-params stringToString       A map from keys to values for jobs with pipeline tasks. (default [])
+      --python-named-params stringToString   A map from keys to values for jobs with Python wheel tasks. (default [])
+      --python-params strings                A list of parameters for jobs with Python tasks.
+      --spark-submit-params strings          A list of parameters for jobs with Spark submit tasks.
+      --sql-params stringToString            A map from keys to values for jobs with SQL tasks. (default [])
+
+Pipeline Flags:
+      --full-refresh strings   List of tables to reset and recompute.
+      --full-refresh-all       Perform a full graph reset and recompute.
+      --refresh strings        List of tables to update.
+      --refresh-all            Perform a full graph update.
+      --validate-only          Perform an update to validate graph correctness.
+
+Flags:
+  -h, --help      help for run
+      --no-wait   Don't wait for the run to complete.
+      --restart   Restart the run if it is already running.
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+      --var strings      set values for variables defined in bundle config. Example: --var="foo=bar"
diff --git a/acceptance/bundle/help/bundle-run/script b/acceptance/bundle/help/bundle-run/script
new file mode 100644
index 000000000..edcf1786a
--- /dev/null
+++ b/acceptance/bundle/help/bundle-run/script
@@ -0,0 +1 @@
+trace $CLI bundle run --help
diff --git a/acceptance/bundle/help/bundle-schema/output.txt b/acceptance/bundle/help/bundle-schema/output.txt
new file mode 100644
index 000000000..8f2983f5b
--- /dev/null
+++ b/acceptance/bundle/help/bundle-schema/output.txt
@@ -0,0 +1,16 @@
+
+>>> $CLI bundle schema --help
+Generate JSON Schema for bundle configuration
+
+Usage:
+  databricks bundle schema [flags]
+
+Flags:
+  -h, --help   help for schema
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+      --var strings      set values for variables defined in bundle config. Example: --var="foo=bar"
diff --git a/acceptance/bundle/help/bundle-schema/script b/acceptance/bundle/help/bundle-schema/script
new file mode 100644
index 000000000..a084fec8e
--- /dev/null
+++ b/acceptance/bundle/help/bundle-schema/script
@@ -0,0 +1 @@
+trace $CLI bundle schema --help
diff --git a/acceptance/bundle/help/bundle-summary/output.txt b/acceptance/bundle/help/bundle-summary/output.txt
new file mode 100644
index 000000000..935c4bdc5
--- /dev/null
+++ b/acceptance/bundle/help/bundle-summary/output.txt
@@ -0,0 +1,17 @@
+
+>>> $CLI bundle summary --help
+Summarize resources deployed by this bundle
+
+Usage:
+  databricks bundle summary [flags]
+
+Flags:
+      --force-pull   Skip local cache and load the state from the remote workspace
+  -h, --help         help for summary
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+      --var strings      set values for variables defined in bundle config. Example: --var="foo=bar"
diff --git a/acceptance/bundle/help/bundle-summary/script b/acceptance/bundle/help/bundle-summary/script
new file mode 100644
index 000000000..967279d86
--- /dev/null
+++ b/acceptance/bundle/help/bundle-summary/script
@@ -0,0 +1 @@
+trace $CLI bundle summary --help
diff --git a/acceptance/bundle/help/bundle-sync/output.txt b/acceptance/bundle/help/bundle-sync/output.txt
new file mode 100644
index 000000000..6588e6978
--- /dev/null
+++ b/acceptance/bundle/help/bundle-sync/output.txt
@@ -0,0 +1,19 @@
+
+>>> $CLI bundle sync --help
+Synchronize bundle tree to the workspace
+
+Usage:
+  databricks bundle sync [flags]
+
+Flags:
+      --full                perform full synchronization (default is incremental)
+  -h, --help                help for sync
+      --interval duration   file system polling interval (for --watch) (default 1s)
+      --output type         type of the output format
+      --watch               watch local file system for changes
+
+Global Flags:
+      --debug            enable debug logging
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+      --var strings      set values for variables defined in bundle config. Example: --var="foo=bar"
diff --git a/acceptance/bundle/help/bundle-sync/script b/acceptance/bundle/help/bundle-sync/script
new file mode 100644
index 000000000..fe1d6c7e3
--- /dev/null
+++ b/acceptance/bundle/help/bundle-sync/script
@@ -0,0 +1 @@
+trace $CLI bundle sync --help
diff --git a/acceptance/bundle/help/bundle-validate/output.txt b/acceptance/bundle/help/bundle-validate/output.txt
new file mode 100644
index 000000000..a0c350faf
--- /dev/null
+++ b/acceptance/bundle/help/bundle-validate/output.txt
@@ -0,0 +1,16 @@
+
+>>> $CLI bundle validate --help
+Validate configuration
+
+Usage:
+  databricks bundle validate [flags]
+
+Flags:
+  -h, --help   help for validate
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+      --var strings      set values for variables defined in bundle config. Example: --var="foo=bar"
diff --git a/acceptance/bundle/help/bundle-validate/script b/acceptance/bundle/help/bundle-validate/script
new file mode 100644
index 000000000..8b8434b2d
--- /dev/null
+++ b/acceptance/bundle/help/bundle-validate/script
@@ -0,0 +1 @@
+trace $CLI bundle validate --help
diff --git a/acceptance/bundle/help/bundle/output.txt b/acceptance/bundle/help/bundle/output.txt
new file mode 100644
index 000000000..e0e2ea47c
--- /dev/null
+++ b/acceptance/bundle/help/bundle/output.txt
@@ -0,0 +1,33 @@
+
+>>> $CLI bundle --help
+Databricks Asset Bundles let you express data/AI/analytics projects as code.
+
+Online documentation: https://docs.databricks.com/en/dev-tools/bundles/index.html
+
+Usage:
+  databricks bundle [command]
+
+Available Commands:
+  deploy      Deploy bundle
+  deployment  Deployment related commands
+  destroy     Destroy deployed bundle resources
+  generate    Generate bundle configuration
+  init        Initialize using a bundle template
+  open        Open a resource in the browser
+  run         Run a job or pipeline update
+  schema      Generate JSON Schema for bundle configuration
+  summary     Summarize resources deployed by this bundle
+  sync        Synchronize bundle tree to the workspace
+  validate    Validate configuration
+
+Flags:
+  -h, --help          help for bundle
+      --var strings   set values for variables defined in bundle config. Example: --var="foo=bar"
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+
+Use "databricks bundle [command] --help" for more information about a command.
diff --git a/acceptance/bundle/help/bundle/script b/acceptance/bundle/help/bundle/script
new file mode 100644
index 000000000..eac116817
--- /dev/null
+++ b/acceptance/bundle/help/bundle/script
@@ -0,0 +1 @@
+trace $CLI bundle --help
diff --git a/cmd/bundle/generate/app.go b/cmd/bundle/generate/app.go
index 819b62b38..9dbd4fe46 100644
--- a/cmd/bundle/generate/app.go
+++ b/cmd/bundle/generate/app.go
@@ -36,8 +36,8 @@ func NewGenerateAppCommand() *cobra.Command {
 	cmd.Flags().StringVar(&appName, "existing-app-name", "", `App name to generate config for`)
 	cmd.MarkFlagRequired("existing-app-name")
 
-	cmd.Flags().StringVarP(&configDir, "config-dir", "d", filepath.Join("resources"), `Directory path where the output bundle config will be stored`)
-	cmd.Flags().StringVarP(&sourceDir, "source-dir", "s", filepath.Join("src", "app"), `Directory path where the app files will be stored`)
+	cmd.Flags().StringVarP(&configDir, "config-dir", "d", "resources", `Directory path where the output bundle config will be stored`)
+	cmd.Flags().StringVarP(&sourceDir, "source-dir", "s", "src/app", `Directory path where the app files will be stored`)
 	cmd.Flags().BoolVarP(&force, "force", "f", false, `Force overwrite existing files in the output directory`)
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {
diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go
index fa3c91b2a..d56d246c2 100644
--- a/cmd/bundle/generate/dashboard.go
+++ b/cmd/bundle/generate/dashboard.go
@@ -441,8 +441,8 @@ func NewGenerateDashboardCommand() *cobra.Command {
 	cmd.Flags().MarkHidden("existing-dashboard-id")
 
 	// Output flags.
-	cmd.Flags().StringVarP(&d.resourceDir, "resource-dir", "d", "./resources", `directory to write the configuration to`)
-	cmd.Flags().StringVarP(&d.dashboardDir, "dashboard-dir", "s", "./src", `directory to write the dashboard representation to`)
+	cmd.Flags().StringVarP(&d.resourceDir, "resource-dir", "d", "resources", `directory to write the configuration to`)
+	cmd.Flags().StringVarP(&d.dashboardDir, "dashboard-dir", "s", "src", `directory to write the dashboard representation to`)
 	cmd.Flags().BoolVarP(&d.force, "force", "f", false, `force overwrite existing files in the output directory`)
 
 	// Exactly one of the lookup flags must be provided.
diff --git a/cmd/bundle/generate/job.go b/cmd/bundle/generate/job.go
index 827d270e5..d97891cd5 100644
--- a/cmd/bundle/generate/job.go
+++ b/cmd/bundle/generate/job.go
@@ -32,13 +32,8 @@ func NewGenerateJobCommand() *cobra.Command {
 	cmd.Flags().Int64Var(&jobId, "existing-job-id", 0, `Job ID of the job to generate config for`)
 	cmd.MarkFlagRequired("existing-job-id")
 
-	wd, err := os.Getwd()
-	if err != nil {
-		wd = "."
-	}
-
-	cmd.Flags().StringVarP(&configDir, "config-dir", "d", filepath.Join(wd, "resources"), `Dir path where the output config will be stored`)
-	cmd.Flags().StringVarP(&sourceDir, "source-dir", "s", filepath.Join(wd, "src"), `Dir path where the downloaded files will be stored`)
+	cmd.Flags().StringVarP(&configDir, "config-dir", "d", "resources", `Dir path where the output config will be stored`)
+	cmd.Flags().StringVarP(&sourceDir, "source-dir", "s", "src", `Dir path where the downloaded files will be stored`)
 	cmd.Flags().BoolVarP(&force, "force", "f", false, `Force overwrite existing files in the output directory`)
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {
diff --git a/cmd/bundle/generate/pipeline.go b/cmd/bundle/generate/pipeline.go
index 863b0b2f7..1d2c345d6 100644
--- a/cmd/bundle/generate/pipeline.go
+++ b/cmd/bundle/generate/pipeline.go
@@ -32,13 +32,8 @@ func NewGeneratePipelineCommand() *cobra.Command {
 	cmd.Flags().StringVar(&pipelineId, "existing-pipeline-id", "", `ID of the pipeline to generate config for`)
 	cmd.MarkFlagRequired("existing-pipeline-id")
 
-	wd, err := os.Getwd()
-	if err != nil {
-		wd = "."
-	}
-
-	cmd.Flags().StringVarP(&configDir, "config-dir", "d", filepath.Join(wd, "resources"), `Dir path where the output config will be stored`)
-	cmd.Flags().StringVarP(&sourceDir, "source-dir", "s", filepath.Join(wd, "src"), `Dir path where the downloaded files will be stored`)
+	cmd.Flags().StringVarP(&configDir, "config-dir", "d", "resources", `Dir path where the output config will be stored`)
+	cmd.Flags().StringVarP(&sourceDir, "source-dir", "s", "src", `Dir path where the downloaded files will be stored`)
 	cmd.Flags().BoolVarP(&force, "force", "f", false, `Force overwrite existing files in the output directory`)
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {

From 50f62692ce119795ea69d8daecc33741a7a332be Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Fri, 17 Jan 2025 16:03:59 +0100
Subject: [PATCH 106/247] Include a materialized copy of built-in templates
 (#2146)

## Changes

Include a materialized copy of built-in templates as reference output.

This updates the output comparison logic to work against an output
directory. The `doComparison` function now always works on real files.
It can now tell apart non-existing files and empty files (e.g., the
`.gitkeep` files in templates).
---
 acceptance/acceptance_test.go                 | 160 +++++++++++-------
 acceptance/bundle/init/dbt-sql/script         |   5 -
 acceptance/bundle/init/dbt-sql/script.cleanup |   1 -
 .../bundle/init/default-python/script.cleanup |   1 -
 acceptance/bundle/init/default-sql/script     |   5 -
 .../bundle/init/default-sql/script.cleanup    |   1 -
 .../{init => templates}/dbt-sql/input.json    |   0
 .../{init => templates}/dbt-sql/output.txt    |   2 +-
 .../dbt-sql/output/my_dbt_sql/.gitignore      |   2 +
 .../my_dbt_sql/.vscode/__builtins__.pyi       |   3 +
 .../output/my_dbt_sql/.vscode/extensions.json |   6 +
 .../output/my_dbt_sql/.vscode/settings.json   |  32 ++++
 .../dbt-sql/output/my_dbt_sql/README.md       | 138 +++++++++++++++
 .../dbt-sql/output/my_dbt_sql/databricks.yml  |  34 ++++
 .../my_dbt_sql/dbt_profiles/profiles.yml      |  38 +++++
 .../dbt-sql/output/my_dbt_sql/dbt_project.yml |  32 ++++
 .../output/my_dbt_sql/profile_template.yml    |  23 +++
 .../output/my_dbt_sql/requirements-dev.txt    |   3 +
 .../my_dbt_sql/resources/my_dbt_sql.job.yml   |  43 +++++
 .../output/my_dbt_sql/src/analyses/.gitkeep   |   0
 .../output/my_dbt_sql/src/macros/.gitkeep     |   0
 .../src/models/example/orders_daily.sql       |  17 ++
 .../src/models/example/orders_raw.sql         |  16 ++
 .../my_dbt_sql/src/models/example/schema.yml  |  21 +++
 .../output/my_dbt_sql/src/seeds/.gitkeep      |   0
 .../output/my_dbt_sql/src/snapshots/.gitkeep  |   0
 .../output/my_dbt_sql/src/tests/.gitkeep      |   0
 acceptance/bundle/templates/dbt-sql/script    |   5 +
 .../default-python/input.json                 |   0
 .../default-python/output.txt                 |   2 +-
 .../output/my_default_python/.gitignore       |   8 +
 .../.vscode/__builtins__.pyi                  |   3 +
 .../my_default_python/.vscode/extensions.json |   7 +
 .../my_default_python/.vscode/settings.json   |  16 ++
 .../output/my_default_python/README.md        |  47 +++++
 .../output/my_default_python/databricks.yml   |  31 ++++
 .../my_default_python/fixtures/.gitkeep       |  22 +++
 .../output/my_default_python/pytest.ini       |   3 +
 .../my_default_python/requirements-dev.txt    |  29 ++++
 .../resources/my_default_python.job.yml       |  49 ++++++
 .../resources/my_default_python.pipeline.yml  |  13 ++
 .../my_default_python/scratch/README.md       |   4 +
 .../scratch/exploration.ipynb                 |  61 +++++++
 .../output/my_default_python/setup.py         |  41 +++++
 .../my_default_python/src/dlt_pipeline.ipynb  |  90 ++++++++++
 .../src/my_default_python/__init__.py         |   1 +
 .../src/my_default_python/main.py             |  25 +++
 .../my_default_python/src/notebook.ipynb      |  75 ++++++++
 .../my_default_python/tests/main_test.py      |   6 +
 .../{init => templates}/default-python/script |   4 +-
 .../bundle/templates/default-sql/.ruff.toml   |   2 +
 .../default-sql/input.json                    |   0
 .../default-sql/output.txt                    |   2 +-
 .../output/my_default_sql/.gitignore          |   2 +
 .../my_default_sql/.vscode/extensions.json    |   7 +
 .../my_default_sql/.vscode/settings.json      |  27 +++
 .../output/my_default_sql/README.md           |  41 +++++
 .../output/my_default_sql/databricks.yml      |  48 ++++++
 .../resources/my_default_sql_sql.job.yml      |  38 +++++
 .../output/my_default_sql/scratch/README.md   |   4 +
 .../my_default_sql/scratch/exploration.ipynb  |  35 ++++
 .../my_default_sql/src/orders_daily.sql       |  21 +++
 .../output/my_default_sql/src/orders_raw.sql  |  19 +++
 .../bundle/templates/default-sql/script       |   5 +
 libs/testdiff/testdiff.go                     |   6 +-
 65 files changed, 1302 insertions(+), 80 deletions(-)
 delete mode 100644 acceptance/bundle/init/dbt-sql/script
 delete mode 100644 acceptance/bundle/init/dbt-sql/script.cleanup
 delete mode 100644 acceptance/bundle/init/default-python/script.cleanup
 delete mode 100644 acceptance/bundle/init/default-sql/script
 delete mode 100644 acceptance/bundle/init/default-sql/script.cleanup
 rename acceptance/bundle/{init => templates}/dbt-sql/input.json (100%)
 rename acceptance/bundle/{init => templates}/dbt-sql/output.txt (92%)
 create mode 100644 acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.gitignore
 create mode 100644 acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.vscode/__builtins__.pyi
 create mode 100644 acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.vscode/extensions.json
 create mode 100644 acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.vscode/settings.json
 create mode 100644 acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/README.md
 create mode 100644 acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml
 create mode 100644 acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_profiles/profiles.yml
 create mode 100644 acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_project.yml
 create mode 100644 acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/profile_template.yml
 create mode 100644 acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/requirements-dev.txt
 create mode 100644 acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml
 create mode 100644 acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/analyses/.gitkeep
 create mode 100644 acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/macros/.gitkeep
 create mode 100644 acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/orders_daily.sql
 create mode 100644 acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/orders_raw.sql
 create mode 100644 acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/schema.yml
 create mode 100644 acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/seeds/.gitkeep
 create mode 100644 acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/snapshots/.gitkeep
 create mode 100644 acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/tests/.gitkeep
 create mode 100644 acceptance/bundle/templates/dbt-sql/script
 rename acceptance/bundle/{init => templates}/default-python/input.json (100%)
 rename acceptance/bundle/{init => templates}/default-python/output.txt (90%)
 create mode 100644 acceptance/bundle/templates/default-python/output/my_default_python/.gitignore
 create mode 100644 acceptance/bundle/templates/default-python/output/my_default_python/.vscode/__builtins__.pyi
 create mode 100644 acceptance/bundle/templates/default-python/output/my_default_python/.vscode/extensions.json
 create mode 100644 acceptance/bundle/templates/default-python/output/my_default_python/.vscode/settings.json
 create mode 100644 acceptance/bundle/templates/default-python/output/my_default_python/README.md
 create mode 100644 acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml
 create mode 100644 acceptance/bundle/templates/default-python/output/my_default_python/fixtures/.gitkeep
 create mode 100644 acceptance/bundle/templates/default-python/output/my_default_python/pytest.ini
 create mode 100644 acceptance/bundle/templates/default-python/output/my_default_python/requirements-dev.txt
 create mode 100644 acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml
 create mode 100644 acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.pipeline.yml
 create mode 100644 acceptance/bundle/templates/default-python/output/my_default_python/scratch/README.md
 create mode 100644 acceptance/bundle/templates/default-python/output/my_default_python/scratch/exploration.ipynb
 create mode 100644 acceptance/bundle/templates/default-python/output/my_default_python/setup.py
 create mode 100644 acceptance/bundle/templates/default-python/output/my_default_python/src/dlt_pipeline.ipynb
 create mode 100644 acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/__init__.py
 create mode 100644 acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/main.py
 create mode 100644 acceptance/bundle/templates/default-python/output/my_default_python/src/notebook.ipynb
 create mode 100644 acceptance/bundle/templates/default-python/output/my_default_python/tests/main_test.py
 rename acceptance/bundle/{init => templates}/default-python/script (73%)
 create mode 100644 acceptance/bundle/templates/default-sql/.ruff.toml
 rename acceptance/bundle/{init => templates}/default-sql/input.json (100%)
 rename acceptance/bundle/{init => templates}/default-sql/output.txt (91%)
 create mode 100644 acceptance/bundle/templates/default-sql/output/my_default_sql/.gitignore
 create mode 100644 acceptance/bundle/templates/default-sql/output/my_default_sql/.vscode/extensions.json
 create mode 100644 acceptance/bundle/templates/default-sql/output/my_default_sql/.vscode/settings.json
 create mode 100644 acceptance/bundle/templates/default-sql/output/my_default_sql/README.md
 create mode 100644 acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml
 create mode 100644 acceptance/bundle/templates/default-sql/output/my_default_sql/resources/my_default_sql_sql.job.yml
 create mode 100644 acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/README.md
 create mode 100644 acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/exploration.ipynb
 create mode 100644 acceptance/bundle/templates/default-sql/output/my_default_sql/src/orders_daily.sql
 create mode 100644 acceptance/bundle/templates/default-sql/output/my_default_sql/src/orders_raw.sql
 create mode 100644 acceptance/bundle/templates/default-sql/script

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index e6eb380a4..2dfe2bce4 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -89,6 +89,7 @@ func TestAccept(t *testing.T) {
 	require.NotNil(t, user)
 	testdiff.PrepareReplacementsUser(t, &repls, *user)
 	testdiff.PrepareReplacementsWorkspaceClient(t, &repls, workspaceClient)
+	testdiff.PrepareReplacementsUUID(t, &repls)
 
 	testDirs := getTests(t)
 	require.NotEmpty(t, testDirs)
@@ -154,70 +155,86 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		require.NoError(t, err)
 		cmd.Env = append(os.Environ(), "GOCOVERDIR="+coverDir)
 	}
+
+	// Write combined output to a file
+	out, err := os.Create(filepath.Join(tmpDir, "output.txt"))
+	require.NoError(t, err)
+	cmd.Stdout = out
+	cmd.Stderr = out
 	cmd.Dir = tmpDir
-	outB, err := cmd.CombinedOutput()
+	err = cmd.Run()
 
-	out := formatOutput(string(outB), err)
-	out = repls.Replace(out)
-	doComparison(t, filepath.Join(dir, "output.txt"), "script output", out)
+	// Include exit code in output (if non-zero)
+	formatOutput(out, err)
+	require.NoError(t, out.Close())
 
-	for key := range outputs {
-		if key == "output.txt" {
-			// handled above
-			continue
-		}
-		pathNew := filepath.Join(tmpDir, key)
-		newValBytes, err := os.ReadFile(pathNew)
-		if err != nil {
-			if errors.Is(err, os.ErrNotExist) {
-				t.Errorf("%s: expected to find this file but could not (%s)", key, tmpDir)
-			} else {
-				t.Errorf("%s: could not read: %s", key, err)
-			}
-			continue
-		}
-		pathExpected := filepath.Join(dir, key)
-		newVal := repls.Replace(string(newValBytes))
-		doComparison(t, pathExpected, pathNew, newVal)
+	// Compare expected outputs
+	for relPath := range outputs {
+		doComparison(t, repls, dir, tmpDir, relPath)
 	}
 
 	// Make sure there are not unaccounted for new files
-	files, err := os.ReadDir(tmpDir)
+	files, err := ListDir(t, tmpDir)
 	require.NoError(t, err)
-
-	for _, f := range files {
-		name := f.Name()
-		if _, ok := inputs[name]; ok {
+	for _, relPath := range files {
+		if _, ok := inputs[relPath]; ok {
 			continue
 		}
-		if _, ok := outputs[name]; ok {
+		if _, ok := outputs[relPath]; ok {
 			continue
 		}
-		t.Errorf("Unexpected output: %s", f)
-		if strings.HasPrefix(name, "out") {
+		if strings.HasPrefix(relPath, "out") {
 			// We have a new file starting with "out"
 			// Show the contents & support overwrite mode for it:
-			pathNew := filepath.Join(tmpDir, name)
-			newVal := testutil.ReadFile(t, pathNew)
-			newVal = repls.Replace(newVal)
-			doComparison(t, filepath.Join(dir, name), filepath.Join(tmpDir, name), newVal)
+			doComparison(t, repls, dir, tmpDir, relPath)
 		}
 	}
 }
 
-func doComparison(t *testing.T, pathExpected, pathNew, valueNew string) {
-	valueNew = testdiff.NormalizeNewlines(valueNew)
-	valueExpected := string(readIfExists(t, pathExpected))
-	valueExpected = testdiff.NormalizeNewlines(valueExpected)
-	testdiff.AssertEqualTexts(t, pathExpected, pathNew, valueExpected, valueNew)
-	if testdiff.OverwriteMode {
-		if valueNew != "" {
-			t.Logf("Overwriting: %s", pathExpected)
-			testutil.WriteFile(t, pathExpected, valueNew)
-		} else {
-			t.Logf("Removing: %s", pathExpected)
-			_ = os.Remove(pathExpected)
+func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string) {
+	pathRef := filepath.Join(dirRef, relPath)
+	pathNew := filepath.Join(dirNew, relPath)
+	bufRef, okRef := readIfExists(t, pathRef)
+	bufNew, okNew := readIfExists(t, pathNew)
+	if !okRef && !okNew {
+		t.Errorf("Both files are missing: %s, %s", pathRef, pathNew)
+		return
+	}
+
+	valueRef := testdiff.NormalizeNewlines(string(bufRef))
+	valueNew := testdiff.NormalizeNewlines(string(bufNew))
+
+	// Apply replacements to the new value only.
+	// The reference value is stored after applying replacements.
+	valueNew = repls.Replace(valueNew)
+
+	// The test did not produce an expected output file.
+	if okRef && !okNew {
+		t.Errorf("Missing output file: %s", relPath)
+		testdiff.AssertEqualTexts(t, pathRef, pathNew, valueRef, valueNew)
+		if testdiff.OverwriteMode {
+			t.Logf("Removing output file: %s", relPath)
+			require.NoError(t, os.Remove(pathRef))
 		}
+		return
+	}
+
+	// The test produced an unexpected output file.
+	if !okRef && okNew {
+		t.Errorf("Unexpected output file: %s", relPath)
+		testdiff.AssertEqualTexts(t, pathRef, pathNew, valueRef, valueNew)
+		if testdiff.OverwriteMode {
+			t.Logf("Writing output file: %s", relPath)
+			testutil.WriteFile(t, pathRef, valueNew)
+		}
+		return
+	}
+
+	// Compare the reference and new values.
+	equal := testdiff.AssertEqualTexts(t, pathRef, pathNew, valueRef, valueNew)
+	if !equal && testdiff.OverwriteMode {
+		t.Logf("Overwriting existing output file: %s", relPath)
+		testutil.WriteFile(t, pathRef, valueNew)
 	}
 }
 
@@ -234,13 +251,13 @@ func readMergedScriptContents(t *testing.T, dir string) string {
 	cleanups := []string{}
 
 	for {
-		x := readIfExists(t, filepath.Join(dir, CleanupScript))
-		if len(x) > 0 {
+		x, ok := readIfExists(t, filepath.Join(dir, CleanupScript))
+		if ok {
 			cleanups = append(cleanups, string(x))
 		}
 
-		x = readIfExists(t, filepath.Join(dir, PrepareScript))
-		if len(x) > 0 {
+		x, ok = readIfExists(t, filepath.Join(dir, PrepareScript))
+		if ok {
 			prepares = append(prepares, string(x))
 		}
 
@@ -316,29 +333,28 @@ func copyFile(src, dst string) error {
 	return err
 }
 
-func formatOutput(out string, err error) string {
+func formatOutput(w io.Writer, err error) {
 	if err == nil {
-		return out
+		return
 	}
 	if exiterr, ok := err.(*exec.ExitError); ok {
 		exitCode := exiterr.ExitCode()
-		out += fmt.Sprintf("\nExit code: %d\n", exitCode)
+		fmt.Fprintf(w, "\nExit code: %d\n", exitCode)
 	} else {
-		out += fmt.Sprintf("\nError: %s\n", err)
+		fmt.Fprintf(w, "\nError: %s\n", err)
 	}
-	return out
 }
 
-func readIfExists(t *testing.T, path string) []byte {
+func readIfExists(t *testing.T, path string) ([]byte, bool) {
 	data, err := os.ReadFile(path)
 	if err == nil {
-		return data
+		return data, true
 	}
 
 	if !errors.Is(err, os.ErrNotExist) {
 		t.Fatalf("%s: %s", path, err)
 	}
-	return []byte{}
+	return []byte{}, false
 }
 
 func CopyDir(src, dst string, inputs, outputs map[string]bool) error {
@@ -353,8 +369,10 @@ func CopyDir(src, dst string, inputs, outputs map[string]bool) error {
 			return err
 		}
 
-		if strings.HasPrefix(name, "out") {
-			outputs[relPath] = true
+		if strings.HasPrefix(relPath, "out") {
+			if !info.IsDir() {
+				outputs[relPath] = true
+			}
 			return nil
 		} else {
 			inputs[relPath] = true
@@ -373,3 +391,25 @@ func CopyDir(src, dst string, inputs, outputs map[string]bool) error {
 		return copyFile(path, destPath)
 	})
 }
+
+func ListDir(t *testing.T, src string) ([]string, error) {
+	var files []string
+	err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			return nil
+		}
+
+		relPath, err := filepath.Rel(src, path)
+		if err != nil {
+			return err
+		}
+
+		files = append(files, relPath)
+		return nil
+	})
+	return files, err
+}
diff --git a/acceptance/bundle/init/dbt-sql/script b/acceptance/bundle/init/dbt-sql/script
deleted file mode 100644
index 5f6efda99..000000000
--- a/acceptance/bundle/init/dbt-sql/script
+++ /dev/null
@@ -1,5 +0,0 @@
-trace $CLI bundle init dbt-sql --config-file ./input.json
-
-cd my_dbt_sql
-trace $CLI bundle validate -t dev
-trace $CLI bundle validate -t prod
diff --git a/acceptance/bundle/init/dbt-sql/script.cleanup b/acceptance/bundle/init/dbt-sql/script.cleanup
deleted file mode 100644
index b717c8d4a..000000000
--- a/acceptance/bundle/init/dbt-sql/script.cleanup
+++ /dev/null
@@ -1 +0,0 @@
-rm -fr my_dbt_sql
diff --git a/acceptance/bundle/init/default-python/script.cleanup b/acceptance/bundle/init/default-python/script.cleanup
deleted file mode 100644
index 4fd2e4aa7..000000000
--- a/acceptance/bundle/init/default-python/script.cleanup
+++ /dev/null
@@ -1 +0,0 @@
-rm -fr my_default_python
diff --git a/acceptance/bundle/init/default-sql/script b/acceptance/bundle/init/default-sql/script
deleted file mode 100644
index 6f7332a39..000000000
--- a/acceptance/bundle/init/default-sql/script
+++ /dev/null
@@ -1,5 +0,0 @@
-trace $CLI bundle init default-sql --config-file ./input.json
-
-cd my_default_sql
-trace $CLI bundle validate -t dev
-trace $CLI bundle validate -t prod
diff --git a/acceptance/bundle/init/default-sql/script.cleanup b/acceptance/bundle/init/default-sql/script.cleanup
deleted file mode 100644
index c46d9ad60..000000000
--- a/acceptance/bundle/init/default-sql/script.cleanup
+++ /dev/null
@@ -1 +0,0 @@
-rm -fr my_default_sql
diff --git a/acceptance/bundle/init/dbt-sql/input.json b/acceptance/bundle/templates/dbt-sql/input.json
similarity index 100%
rename from acceptance/bundle/init/dbt-sql/input.json
rename to acceptance/bundle/templates/dbt-sql/input.json
diff --git a/acceptance/bundle/init/dbt-sql/output.txt b/acceptance/bundle/templates/dbt-sql/output.txt
similarity index 92%
rename from acceptance/bundle/init/dbt-sql/output.txt
rename to acceptance/bundle/templates/dbt-sql/output.txt
index 4f6f83693..972c7e152 100644
--- a/acceptance/bundle/init/dbt-sql/output.txt
+++ b/acceptance/bundle/templates/dbt-sql/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle init dbt-sql --config-file ./input.json
+>>> $CLI bundle init dbt-sql --config-file ./input.json --output-dir output
 
 Welcome to the dbt template for Databricks Asset Bundles!
 
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.gitignore b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.gitignore
new file mode 100644
index 000000000..de811f118
--- /dev/null
+++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.gitignore
@@ -0,0 +1,2 @@
+
+.databricks
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.vscode/__builtins__.pyi b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.vscode/__builtins__.pyi
new file mode 100644
index 000000000..0edd5181b
--- /dev/null
+++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.vscode/__builtins__.pyi
@@ -0,0 +1,3 @@
+# Typings for Pylance in Visual Studio Code
+# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md
+from databricks.sdk.runtime import *
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.vscode/extensions.json b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.vscode/extensions.json
new file mode 100644
index 000000000..28fe943fd
--- /dev/null
+++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.vscode/extensions.json
@@ -0,0 +1,6 @@
+{
+    "recommendations": [
+        "redhat.vscode-yaml",
+        "innoverio.vscode-dbt-power-user",
+    ]
+}
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.vscode/settings.json b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.vscode/settings.json
new file mode 100644
index 000000000..e8dcd1a83
--- /dev/null
+++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.vscode/settings.json
@@ -0,0 +1,32 @@
+{
+    "python.analysis.stubPath": ".vscode",
+    "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\<codecell\\>|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])",
+    "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------",
+    "python.testing.pytestArgs": [
+        "."
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true,
+    "python.analysis.extraPaths": ["src"],
+    "files.exclude": {
+        "**/*.egg-info": true,
+        "**/__pycache__": true,
+        ".pytest_cache": true,
+    },
+    "python.envFile": "${workspaceFolder}/.databricks/.databricks.env",
+    "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python",
+    "sqltools.connections": [
+        {
+            "connectionMethod": "VS Code Extension (beta)",
+            "catalog": "hive_metastore",
+            "previewLimit": 50,
+            "driver": "Databricks",
+            "name": "databricks",
+            "path": "/sql/2.0/warehouses/f00dcafe"
+        }
+    ],
+    "sqltools.autoConnectTo": "",
+    "[jinja-sql]": {
+        "editor.defaultFormatter": "innoverio.vscode-dbt-power-user"
+    }
+}
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/README.md b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/README.md
new file mode 100644
index 000000000..756a2eda4
--- /dev/null
+++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/README.md
@@ -0,0 +1,138 @@
+# my_dbt_sql
+
+The 'my_dbt_sql' project was generated by using the dbt template for
+Databricks Asset Bundles. It follows the standard dbt project structure
+and has an additional `resources` directory to define Databricks resources such as jobs
+that run dbt models.
+
+* Learn more about dbt and its standard project structure here: https://docs.getdbt.com/docs/build/projects.
+* Learn more about Databricks Asset Bundles here: https://docs.databricks.com/en/dev-tools/bundles/index.html
+
+The remainder of this file includes instructions for local development (using dbt)
+and deployment to production (using Databricks Asset Bundles).
+
+## Development setup
+
+1. Install the Databricks CLI from https://docs.databricks.com/dev-tools/cli/databricks-cli.html
+
+2. Authenticate to your Databricks workspace, if you have not done so already:
+    ```
+    $ databricks configure
+    ```
+
+3. Install dbt
+
+   To install dbt, you need a recent version of Python. For the instructions below,
+   we assume `python3` refers to the Python version you want to use. On some systems,
+   you may need to refer to a different Python version, e.g. `python` or `/usr/bin/python`.
+
+   Run these instructions from the `my_dbt_sql` directory. We recommend making
+   use of a Python virtual environment and installing dbt as follows:
+
+   ```
+   $ python3 -m venv .venv
+   $ . .venv/bin/activate
+   $ pip install -r requirements-dev.txt
+   ```
+
+4. Initialize your dbt profile
+
+   Use `dbt init` to initialize your profile.
+
+   ```
+   $ dbt init
+   ```
+
+   Note that dbt authentication uses personal access tokens by default
+   (see https://docs.databricks.com/dev-tools/auth/pat.html).
+   You can use OAuth as an alternative, but this currently requires manual configuration.
+   See https://github.com/databricks/dbt-databricks/blob/main/docs/oauth.md
+   for general instructions, or https://community.databricks.com/t5/technical-blog/using-dbt-core-with-oauth-on-azure-databricks/ba-p/46605
+   for advice on setting up OAuth for Azure Databricks.
+
+   To setup up additional profiles, such as a 'prod' profile,
+   see https://docs.getdbt.com/docs/core/connect-data-platform/connection-profiles.
+
+5. Activate dbt so it can be used from the terminal
+
+   ```
+   $ . .venv/bin/activate
+    ```
+
+## Local development with dbt
+
+Use `dbt` to [run this project locally using a SQL warehouse](https://docs.databricks.com/partners/prep/dbt.html):
+
+```
+$ dbt seed
+$ dbt run
+```
+
+(Did you get an error that the dbt command could not be found? You may need
+to try the last step from the development setup above to re-activate
+your Python virtual environment!)
+
+
+To just evaluate a single model defined in a file called orders.sql, use:
+
+```
+$ dbt run --model orders
+```
+
+Use `dbt test` to run tests generated from yml files such as `models/schema.yml`
+and any SQL tests from `tests/`
+
+```
+$ dbt test
+```
+
+## Production setup
+
+Your production dbt profiles are defined in dbt_profiles/profiles.yml.
+These profiles define the default catalog, schema, and any other
+target-specific settings. Read more about dbt profiles on Databricks at
+https://docs.databricks.com/en/workflows/jobs/how-to/use-dbt-in-workflows.html#advanced-run-dbt-with-a-custom-profile.
+
+The target workspaces for staging and prod are defined in databricks.yml.
+You can manually deploy based on these configurations (see below).
+Or you can use CI/CD to automate deployment. See
+https://docs.databricks.com/dev-tools/bundles/ci-cd.html for documentation
+on CI/CD setup.
+
+## Manually deploying to Databricks with Databricks Asset Bundles
+
+Databricks Asset Bundles can be used to deploy to Databricks and to execute
+dbt commands as a job using Databricks Workflows. See
+https://docs.databricks.com/dev-tools/bundles/index.html to learn more.
+
+Use the Databricks CLI to deploy a development copy of this project to a workspace:
+
+```
+$ databricks bundle deploy --target dev
+```
+
+(Note that "dev" is the default target, so the `--target` parameter
+is optional here.)
+
+This deploys everything that's defined for this project.
+For example, the default template would deploy a job called
+`[dev yourname] my_dbt_sql_job` to your workspace.
+You can find that job by opening your workpace and clicking on **Workflows**.
+
+You can also deploy to your production target directly from the command-line.
+The warehouse, catalog, and schema for that target are configured in databricks.yml.
+When deploying to this target, note that the default job at resources/my_dbt_sql.job.yml
+has a schedule set that runs every day. The schedule is paused when deploying in development mode
+(see https://docs.databricks.com/dev-tools/bundles/deployment-modes.html).
+
+To deploy a production copy, type:
+
+```
+$ databricks bundle deploy --target prod
+```
+
+## IDE support
+
+Optionally, install developer tools such as the Databricks extension for Visual Studio Code from
+https://docs.databricks.com/dev-tools/vscode-ext.html. Third-party extensions
+related to dbt may further enhance your dbt development experience!
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml
new file mode 100644
index 000000000..1962bc543
--- /dev/null
+++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml
@@ -0,0 +1,34 @@
+# This file defines the structure of this project and how it is deployed
+# to production using Databricks Asset Bundles.
+# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
+bundle:
+  name: my_dbt_sql
+  uuid: <UUID>
+
+include:
+  - resources/*.yml
+
+# Deployment targets.
+# The default schema, catalog, etc. for dbt are defined in dbt_profiles/profiles.yml
+targets:
+  dev:
+    default: true
+    # The default target uses 'mode: development' to create a development copy.
+    # - Deployed resources get prefixed with '[dev my_user_name]'
+    # - Any job schedules and triggers are paused by default.
+    # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html.
+    mode: development
+    workspace:
+      host: $DATABRICKS_URL
+
+  prod:
+    mode: production
+    workspace:
+      host: $DATABRICKS_URL
+      # We explicitly specify /Workspace/Users/$USERNAME to make sure we only have a single copy.
+      root_path: /Workspace/Users/$USERNAME/.bundle/${bundle.name}/${bundle.target}
+    permissions:
+      - user_name: $USERNAME
+        level: CAN_MANAGE
+    run_as:
+      user_name: $USERNAME
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_profiles/profiles.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_profiles/profiles.yml
new file mode 100644
index 000000000..fdaf30dda
--- /dev/null
+++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_profiles/profiles.yml
@@ -0,0 +1,38 @@
+
+# This file defines dbt profiles for deployed dbt jobs.
+my_dbt_sql:
+   target: dev # default target
+   outputs:
+
+    # Doing local development with the dbt CLI?
+    # Then you should create your own profile in your .dbt/profiles.yml using 'dbt init'
+    # (See README.md)
+
+    # The default target when deployed with the Databricks CLI
+    # N.B. when you use dbt from the command line, it uses the profile from .dbt/profiles.yml
+    dev:
+      type: databricks
+      method: http
+      catalog: main
+      schema: "{{ var('dev_schema') }}"
+
+      http_path: /sql/2.0/warehouses/f00dcafe
+
+      # The workspace host / token are provided by Databricks
+      # see databricks.yml for the workspace host used for 'dev'
+      host: "{{ env_var('DBT_HOST') }}"
+      token: "{{ env_var('DBT_ACCESS_TOKEN') }}"
+
+    # The production target when deployed with the Databricks CLI
+    prod:
+      type: databricks
+      method: http
+      catalog: main
+      schema: default
+
+      http_path: /sql/2.0/warehouses/f00dcafe
+
+      # The workspace host / token are provided by Databricks
+      # see databricks.yml for the workspace host used for 'prod'
+      host: "{{ env_var('DBT_HOST') }}"
+      token: "{{ env_var('DBT_ACCESS_TOKEN') }}"
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_project.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_project.yml
new file mode 100644
index 000000000..4218640d8
--- /dev/null
+++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_project.yml
@@ -0,0 +1,32 @@
+name: 'my_dbt_sql'
+version: '1.0.0'
+config-version: 2
+
+# This setting configures which "profile" dbt uses for this project.
+profile: 'my_dbt_sql'
+
+# These configurations specify where dbt should look for different types of files.
+# For Databricks asset bundles, we put everything in src, as you may have
+# non-dbt resources in your project.
+model-paths: ["src/models"]
+analysis-paths: ["src/analyses"]
+test-paths: ["src/tests"]
+seed-paths: ["src/seeds"]
+macro-paths: ["src/macros"]
+snapshot-paths: ["src/snapshots"]
+
+clean-targets:         # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
+
+# Configuring models
+# Full documentation: https://docs.getdbt.com/docs/configuring-models
+
+# In this example config, we tell dbt to build all models in the example/
+# directory as views by default. These settings can be overridden in the
+# individual model files using the `{{ config(...) }}` macro.
+models:
+  my_dbt_sql:
+    # Config indicated by + and applies to all files under models/example/
+    example:
+      +materialized: view
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/profile_template.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/profile_template.yml
new file mode 100644
index 000000000..5e0f0fc29
--- /dev/null
+++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/profile_template.yml
@@ -0,0 +1,23 @@
+# This file defines prompts with defaults for dbt initializaton.
+# It is used when the `dbt init` command is invoked.
+#
+fixed:
+  type: databricks
+prompts:
+  host:
+    default: $DATABRICKS_HOST
+  token:
+    hint: 'personal access token to use, dapiXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
+    hide_input: true
+  http_path:
+    hint: 'HTTP path of SQL warehouse to use'
+    default: /sql/2.0/warehouses/f00dcafe
+  catalog:
+    hint: 'initial catalog'
+    default: main
+  schema:
+    hint: 'personal schema where dbt will build objects during development, example: $USERNAME'
+  threads:
+    hint: 'threads to use during development, 1 or more'
+    type: 'int'
+    default: 4
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/requirements-dev.txt b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/requirements-dev.txt
new file mode 100644
index 000000000..e6b861203
--- /dev/null
+++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/requirements-dev.txt
@@ -0,0 +1,3 @@
+## requirements-dev.txt: dependencies for local development.
+
+dbt-databricks>=1.8.0,<2.0.0
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml
new file mode 100644
index 000000000..d52f8ed50
--- /dev/null
+++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml
@@ -0,0 +1,43 @@
+resources:
+  jobs:
+    my_dbt_sql_job:
+      name: my_dbt_sql_job
+
+      trigger:
+        # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger
+        periodic:
+          interval: 1
+          unit: DAYS
+
+      email_notifications:
+        on_failure:
+          - $USERNAME
+
+
+      tasks:
+        - task_key: dbt
+
+          dbt_task:
+            project_directory: ../
+            # The default schema, catalog, etc. are defined in ../dbt_profiles/profiles.yml
+            profiles_directory: dbt_profiles/
+            commands:
+            # The dbt commands to run (see also dbt_profiles/profiles.yml; dev_schema is used in the dev profile)
+            - 'dbt deps --target=${bundle.target}'
+            - 'dbt seed --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"'
+            - 'dbt run --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"'
+
+          libraries:
+          - pypi:
+              package: dbt-databricks>=1.8.0,<2.0.0
+
+          new_cluster:
+            spark_version: 15.4.x-scala2.12
+            node_type_id: i3.xlarge
+            data_security_mode: SINGLE_USER
+            num_workers: 0
+            spark_conf:
+                spark.master: "local[*, 4]"
+                spark.databricks.cluster.profile: singleNode
+            custom_tags:
+              ResourceClass: SingleNode
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/analyses/.gitkeep b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/analyses/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/macros/.gitkeep b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/macros/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/orders_daily.sql b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/orders_daily.sql
new file mode 100644
index 000000000..e32736ceb
--- /dev/null
+++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/orders_daily.sql
@@ -0,0 +1,17 @@
+
+-- This model file defines a materialized view called 'orders_daily'
+--
+-- Read more about materialized at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables
+-- Current limitation: a "full refresh" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561.
+{{ config(materialized = 'materialized_view') }}
+
+select order_date, count(*) AS number_of_orders
+
+from {{ ref('orders_raw') }}
+
+-- During development, only process a smaller range of data
+{% if target.name != 'prod' %}
+where order_date >= '2019-08-01' and order_date < '2019-09-01'
+{% endif %}
+
+group by order_date
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/orders_raw.sql b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/orders_raw.sql
new file mode 100644
index 000000000..8faf8f38b
--- /dev/null
+++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/orders_raw.sql
@@ -0,0 +1,16 @@
+-- This model file defines a streaming table called 'orders_raw'
+--
+-- The streaming table below ingests all JSON files in /databricks-datasets/retail-org/sales_orders/
+-- Read more about streaming tables at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables
+-- Current limitation: a "full refresh" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561.
+{{ config(materialized = 'streaming_table') }}
+
+select
+  customer_name,
+  date(timestamp(from_unixtime(try_cast(order_datetime as bigint)))) as order_date,
+  order_number
+from stream read_files(
+  "/databricks-datasets/retail-org/sales_orders/",
+  format => "json",
+  header => true
+)
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/schema.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/schema.yml
new file mode 100644
index 000000000..c64f1bfce
--- /dev/null
+++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/schema.yml
@@ -0,0 +1,21 @@
+
+version: 2
+
+models:
+  - name: orders_raw
+    description: "Raw ingested orders"
+    columns:
+      - name: customer_name
+        description: "The name of a customer"
+        data_tests:
+          - unique
+          - not_null
+
+  - name: orders_daily
+    description: "Number of orders by day"
+    columns:
+      - name: order_date
+        description: "The date on which orders took place"
+        data_tests:
+          - unique
+          - not_null
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/seeds/.gitkeep b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/seeds/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/snapshots/.gitkeep b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/snapshots/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/tests/.gitkeep b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/tests/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/acceptance/bundle/templates/dbt-sql/script b/acceptance/bundle/templates/dbt-sql/script
new file mode 100644
index 000000000..c4ca817fe
--- /dev/null
+++ b/acceptance/bundle/templates/dbt-sql/script
@@ -0,0 +1,5 @@
+trace $CLI bundle init dbt-sql --config-file ./input.json --output-dir output
+
+cd output/my_dbt_sql
+trace $CLI bundle validate -t dev
+trace $CLI bundle validate -t prod
diff --git a/acceptance/bundle/init/default-python/input.json b/acceptance/bundle/templates/default-python/input.json
similarity index 100%
rename from acceptance/bundle/init/default-python/input.json
rename to acceptance/bundle/templates/default-python/input.json
diff --git a/acceptance/bundle/init/default-python/output.txt b/acceptance/bundle/templates/default-python/output.txt
similarity index 90%
rename from acceptance/bundle/init/default-python/output.txt
rename to acceptance/bundle/templates/default-python/output.txt
index a02a7d41b..5493ac2cf 100644
--- a/acceptance/bundle/init/default-python/output.txt
+++ b/acceptance/bundle/templates/default-python/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle init default-python --config-file ./input.json
+>>> $CLI bundle init default-python --config-file ./input.json --output-dir output
 
 Welcome to the default Python template for Databricks Asset Bundles!
 Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): $DATABRICKS_URL
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/.gitignore b/acceptance/bundle/templates/default-python/output/my_default_python/.gitignore
new file mode 100644
index 000000000..0dab7f499
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/.gitignore
@@ -0,0 +1,8 @@
+.databricks/
+build/
+dist/
+__pycache__/
+*.egg-info
+.venv/
+scratch/**
+!scratch/README.md
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/.vscode/__builtins__.pyi b/acceptance/bundle/templates/default-python/output/my_default_python/.vscode/__builtins__.pyi
new file mode 100644
index 000000000..0edd5181b
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/.vscode/__builtins__.pyi
@@ -0,0 +1,3 @@
+# Typings for Pylance in Visual Studio Code
+# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md
+from databricks.sdk.runtime import *
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/.vscode/extensions.json b/acceptance/bundle/templates/default-python/output/my_default_python/.vscode/extensions.json
new file mode 100644
index 000000000..5d15eba36
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/.vscode/extensions.json
@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "databricks.databricks",
+        "ms-python.vscode-pylance",
+        "redhat.vscode-yaml"
+    ]
+}
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/.vscode/settings.json b/acceptance/bundle/templates/default-python/output/my_default_python/.vscode/settings.json
new file mode 100644
index 000000000..8ee87c30d
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/.vscode/settings.json
@@ -0,0 +1,16 @@
+{
+    "python.analysis.stubPath": ".vscode",
+    "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\<codecell\\>|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])",
+    "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------",
+    "python.testing.pytestArgs": [
+        "."
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true,
+    "python.analysis.extraPaths": ["src"],
+    "files.exclude": {
+        "**/*.egg-info": true,
+        "**/__pycache__": true,
+        ".pytest_cache": true,
+    },
+}
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/README.md b/acceptance/bundle/templates/default-python/output/my_default_python/README.md
new file mode 100644
index 000000000..97d7d7949
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/README.md
@@ -0,0 +1,47 @@
+# my_default_python
+
+The 'my_default_python' project was generated by using the default-python template.
+
+## Getting started
+
+1. Install the Databricks CLI from https://docs.databricks.com/dev-tools/cli/databricks-cli.html
+
+2. Authenticate to your Databricks workspace, if you have not done so already:
+    ```
+    $ databricks configure
+    ```
+
+3. To deploy a development copy of this project, type:
+    ```
+    $ databricks bundle deploy --target dev
+    ```
+    (Note that "dev" is the default target, so the `--target` parameter
+    is optional here.)
+
+    This deploys everything that's defined for this project.
+    For example, the default template would deploy a job called
+    `[dev yourname] my_default_python_job` to your workspace.
+    You can find that job by opening your workpace and clicking on **Workflows**.
+
+4. Similarly, to deploy a production copy, type:
+   ```
+   $ databricks bundle deploy --target prod
+   ```
+
+   Note that the default job from the template has a schedule that runs every day
+   (defined in resources/my_default_python.job.yml). The schedule
+   is paused when deploying in development mode (see
+   https://docs.databricks.com/dev-tools/bundles/deployment-modes.html).
+
+5. To run a job or pipeline, use the "run" command:
+   ```
+   $ databricks bundle run
+   ```
+
+6. Optionally, install developer tools such as the Databricks extension for Visual Studio Code from
+   https://docs.databricks.com/dev-tools/vscode-ext.html. Or read the "getting started" documentation for
+   **Databricks Connect** for instructions on running the included Python code from a different IDE.
+
+7. For documentation on the Databricks asset bundles format used
+   for this project, and for CI/CD configuration, see
+   https://docs.databricks.com/dev-tools/bundles/index.html.
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml b/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml
new file mode 100644
index 000000000..9deca9cf5
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml
@@ -0,0 +1,31 @@
+# This is a Databricks asset bundle definition for my_default_python.
+# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
+bundle:
+  name: my_default_python
+  uuid: <UUID>
+
+include:
+  - resources/*.yml
+
+targets:
+  dev:
+    # The default target uses 'mode: development' to create a development copy.
+    # - Deployed resources get prefixed with '[dev my_user_name]'
+    # - Any job schedules and triggers are paused by default.
+    # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html.
+    mode: development
+    default: true
+    workspace:
+      host: $DATABRICKS_URL
+
+  prod:
+    mode: production
+    workspace:
+      host: $DATABRICKS_URL
+      # We explicitly specify /Workspace/Users/$USERNAME to make sure we only have a single copy.
+      root_path: /Workspace/Users/$USERNAME/.bundle/${bundle.name}/${bundle.target}
+    permissions:
+      - user_name: $USERNAME
+        level: CAN_MANAGE
+    run_as:
+      user_name: $USERNAME
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/fixtures/.gitkeep b/acceptance/bundle/templates/default-python/output/my_default_python/fixtures/.gitkeep
new file mode 100644
index 000000000..fa25d2745
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/fixtures/.gitkeep
@@ -0,0 +1,22 @@
+# Fixtures
+
+This folder is reserved for fixtures, such as CSV files.
+
+Below is an example of how to load fixtures as a data frame:
+
+```
+import pandas as pd
+import os
+
+def get_absolute_path(*relative_parts):
+    if 'dbutils' in globals():
+        base_dir = os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) # type: ignore
+        path = os.path.normpath(os.path.join(base_dir, *relative_parts))
+        return path if path.startswith("/Workspace") else "/Workspace" + path
+    else:
+        return os.path.join(*relative_parts)
+
+csv_file = get_absolute_path("..", "fixtures", "mycsv.csv")
+df = pd.read_csv(csv_file)
+display(df)
+```
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/pytest.ini b/acceptance/bundle/templates/default-python/output/my_default_python/pytest.ini
new file mode 100644
index 000000000..80432c220
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/pytest.ini
@@ -0,0 +1,3 @@
+[pytest]
+testpaths = tests
+pythonpath = src
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/requirements-dev.txt b/acceptance/bundle/templates/default-python/output/my_default_python/requirements-dev.txt
new file mode 100644
index 000000000..0ffbf6aed
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/requirements-dev.txt
@@ -0,0 +1,29 @@
+## requirements-dev.txt: dependencies for local development.
+##
+## For defining dependencies used by jobs in Databricks Workflows, see
+## https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
+
+## Add code completion support for DLT
+databricks-dlt
+
+## pytest is the default package used for testing
+pytest
+
+## Dependencies for building wheel files
+setuptools
+wheel
+
+## databricks-connect can be used to run parts of this project locally.
+## See https://docs.databricks.com/dev-tools/databricks-connect.html.
+##
+## databricks-connect is automatically installed if you're using Databricks
+## extension for Visual Studio Code
+## (https://docs.databricks.com/dev-tools/vscode-ext/dev-tasks/databricks-connect.html).
+##
+## To manually install databricks-connect, either follow the instructions
+## at https://docs.databricks.com/dev-tools/databricks-connect.html
+## to install the package system-wide. Or uncomment the line below to install a
+## version of db-connect that corresponds to the Databricks Runtime version used
+## for this project.
+#
+# databricks-connect>=15.4,<15.5
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml b/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml
new file mode 100644
index 000000000..e6148a4ad
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml
@@ -0,0 +1,49 @@
+# The main job for my_default_python.
+resources:
+  jobs:
+    my_default_python_job:
+      name: my_default_python_job
+
+      trigger:
+        # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger
+        periodic:
+          interval: 1
+          unit: DAYS
+
+      email_notifications:
+        on_failure:
+          - $USERNAME
+
+      tasks:
+        - task_key: notebook_task
+          job_cluster_key: job_cluster
+          notebook_task:
+            notebook_path: ../src/notebook.ipynb
+        
+        - task_key: refresh_pipeline
+          depends_on:
+            - task_key: notebook_task
+          pipeline_task:
+            pipeline_id: ${resources.pipelines.my_default_python_pipeline.id}
+        
+        - task_key: main_task
+          depends_on:
+            - task_key: refresh_pipeline
+          job_cluster_key: job_cluster
+          python_wheel_task:
+            package_name: my_default_python
+            entry_point: main
+          libraries:
+            # By default we just include the .whl file generated for the my_default_python package.
+            # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
+            # for more information on how to add other libraries.
+            - whl: ../dist/*.whl
+
+      job_clusters:
+        - job_cluster_key: job_cluster
+          new_cluster:
+            spark_version: 15.4.x-scala2.12
+            node_type_id: i3.xlarge
+            autoscale:
+                min_workers: 1
+                max_workers: 4
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.pipeline.yml b/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.pipeline.yml
new file mode 100644
index 000000000..f9e083f4f
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.pipeline.yml
@@ -0,0 +1,13 @@
+# The main pipeline for my_default_python
+resources:
+  pipelines:
+    my_default_python_pipeline:
+      name: my_default_python_pipeline
+      catalog: main
+      target: my_default_python_${bundle.target}
+      libraries:
+        - notebook:
+            path: ../src/dlt_pipeline.ipynb
+
+      configuration:
+        bundle.sourcePath: ${workspace.file_path}/src
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/scratch/README.md b/acceptance/bundle/templates/default-python/output/my_default_python/scratch/README.md
new file mode 100644
index 000000000..e6cfb81b4
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/scratch/README.md
@@ -0,0 +1,4 @@
+# scratch
+
+This folder is reserved for personal, exploratory notebooks.
+By default these are not committed to Git, as 'scratch' is listed in .gitignore.
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/scratch/exploration.ipynb b/acceptance/bundle/templates/default-python/output/my_default_python/scratch/exploration.ipynb
new file mode 100644
index 000000000..3b2fef4b4
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/scratch/exploration.ipynb
@@ -0,0 +1,61 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "<UUID>",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "\n",
+    "sys.path.append(\"../src\")\n",
+    "from my_default_python import main\n",
+    "\n",
+    "main.get_taxis(spark).show(10)"
+   ]
+  }
+ ],
+ "metadata": {
+  "application/vnd.databricks.v1+notebook": {
+   "dashboards": [],
+   "language": "python",
+   "notebookMetadata": {
+    "pythonIndentUnit": 2
+   },
+   "notebookName": "ipynb-notebook",
+   "widgets": {}
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/setup.py b/acceptance/bundle/templates/default-python/output/my_default_python/setup.py
new file mode 100644
index 000000000..84b24ecb8
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/setup.py
@@ -0,0 +1,41 @@
+"""
+setup.py configuration script describing how to build and package this project.
+
+This file is primarily used by the setuptools library and typically should not
+be executed directly. See README.md for how to deploy, test, and run
+the my_default_python project.
+"""
+
+from setuptools import setup, find_packages
+
+import sys
+
+sys.path.append("./src")
+
+import datetime
+import my_default_python
+
+local_version = datetime.datetime.utcnow().strftime("%Y%m%d.%H%M%S")
+
+setup(
+    name="my_default_python",
+    # We use timestamp as Local version identifier (https://peps.python.org/pep-0440/#local-version-identifiers.)
+    # to ensure that changes to wheel package are picked up when used on all-purpose clusters
+    version=my_default_python.__version__ + "+" + local_version,
+    url="https://databricks.com",
+    author="$USERNAME",
+    description="wheel file based on my_default_python/src",
+    packages=find_packages(where="./src"),
+    package_dir={"": "src"},
+    entry_points={
+        "packages": [
+            "main=my_default_python.main:main",
+        ],
+    },
+    install_requires=[
+        # Dependencies in case the output wheel file is used as a library dependency.
+        # For defining dependencies, when this package is used in Databricks, see:
+        # https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
+        "setuptools"
+    ],
+)
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/src/dlt_pipeline.ipynb b/acceptance/bundle/templates/default-python/output/my_default_python/src/dlt_pipeline.ipynb
new file mode 100644
index 000000000..36e993af7
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/src/dlt_pipeline.ipynb
@@ -0,0 +1,90 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "<UUID>",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "source": [
+    "# DLT pipeline\n",
+    "\n",
+    "This Delta Live Tables (DLT) definition is executed using a pipeline defined in resources/my_default_python.pipeline.yml."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "<UUID>",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Import DLT and src/my_default_python\n",
+    "import dlt\n",
+    "import sys\n",
+    "\n",
+    "sys.path.append(spark.conf.get(\"bundle.sourcePath\", \".\"))\n",
+    "from pyspark.sql.functions import expr\n",
+    "from my_default_python import main"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "<UUID>",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "@dlt.view\n",
+    "def taxi_raw():\n",
+    "    return main.get_taxis(spark)\n",
+    "\n",
+    "\n",
+    "@dlt.table\n",
+    "def filtered_taxis():\n",
+    "    return dlt.read(\"taxi_raw\").filter(expr(\"fare_amount < 30\"))"
+   ]
+  }
+ ],
+ "metadata": {
+  "application/vnd.databricks.v1+notebook": {
+   "dashboards": [],
+   "language": "python",
+   "notebookMetadata": {
+    "pythonIndentUnit": 2
+   },
+   "notebookName": "dlt_pipeline",
+   "widgets": {}
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/__init__.py b/acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/__init__.py
new file mode 100644
index 000000000..f102a9cad
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/__init__.py
@@ -0,0 +1 @@
+__version__ = "0.0.1"
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/main.py b/acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/main.py
new file mode 100644
index 000000000..5ae344c7e
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/main.py
@@ -0,0 +1,25 @@
+from pyspark.sql import SparkSession, DataFrame
+
+
+def get_taxis(spark: SparkSession) -> DataFrame:
+    return spark.read.table("samples.nyctaxi.trips")
+
+
+# Create a new Databricks Connect session. If this fails,
+# check that you have configured Databricks Connect correctly.
+# See https://docs.databricks.com/dev-tools/databricks-connect.html.
+def get_spark() -> SparkSession:
+    try:
+        from databricks.connect import DatabricksSession
+
+        return DatabricksSession.builder.getOrCreate()
+    except ImportError:
+        return SparkSession.builder.getOrCreate()
+
+
+def main():
+    get_taxis(get_spark()).show(5)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/src/notebook.ipynb b/acceptance/bundle/templates/default-python/output/my_default_python/src/notebook.ipynb
new file mode 100644
index 000000000..0d560443b
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/src/notebook.ipynb
@@ -0,0 +1,75 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "<UUID>",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "source": [
+    "# Default notebook\n",
+    "\n",
+    "This default notebook is executed using Databricks Workflows as defined in resources/my_default_python.job.yml."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "<UUID>",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from my_default_python import main\n",
+    "\n",
+    "main.get_taxis(spark).show(10)"
+   ]
+  }
+ ],
+ "metadata": {
+  "application/vnd.databricks.v1+notebook": {
+   "dashboards": [],
+   "language": "python",
+   "notebookMetadata": {
+    "pythonIndentUnit": 2
+   },
+   "notebookName": "notebook",
+   "widgets": {}
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/tests/main_test.py b/acceptance/bundle/templates/default-python/output/my_default_python/tests/main_test.py
new file mode 100644
index 000000000..dc449154a
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/tests/main_test.py
@@ -0,0 +1,6 @@
+from my_default_python.main import get_taxis, get_spark
+
+
+def test_main():
+    taxis = get_taxis(get_spark())
+    assert taxis.count() > 5
diff --git a/acceptance/bundle/init/default-python/script b/acceptance/bundle/templates/default-python/script
similarity index 73%
rename from acceptance/bundle/init/default-python/script
rename to acceptance/bundle/templates/default-python/script
index 84834ce3f..b11a7ea21 100644
--- a/acceptance/bundle/init/default-python/script
+++ b/acceptance/bundle/templates/default-python/script
@@ -1,5 +1,5 @@
-trace $CLI bundle init default-python --config-file ./input.json
+trace $CLI bundle init default-python --config-file ./input.json --output-dir output
 
-cd my_default_python
+cd output/my_default_python
 trace $CLI bundle validate -t dev
 trace $CLI bundle validate -t prod
diff --git a/acceptance/bundle/templates/default-sql/.ruff.toml b/acceptance/bundle/templates/default-sql/.ruff.toml
new file mode 100644
index 000000000..43f86042e
--- /dev/null
+++ b/acceptance/bundle/templates/default-sql/.ruff.toml
@@ -0,0 +1,2 @@
+[format]
+exclude = ["*.ipynb"]
diff --git a/acceptance/bundle/init/default-sql/input.json b/acceptance/bundle/templates/default-sql/input.json
similarity index 100%
rename from acceptance/bundle/init/default-sql/input.json
rename to acceptance/bundle/templates/default-sql/input.json
diff --git a/acceptance/bundle/init/default-sql/output.txt b/acceptance/bundle/templates/default-sql/output.txt
similarity index 91%
rename from acceptance/bundle/init/default-sql/output.txt
rename to acceptance/bundle/templates/default-sql/output.txt
index ff107eae0..fe0139093 100644
--- a/acceptance/bundle/init/default-sql/output.txt
+++ b/acceptance/bundle/templates/default-sql/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle init default-sql --config-file ./input.json
+>>> $CLI bundle init default-sql --config-file ./input.json --output-dir output
 
 Welcome to the default SQL template for Databricks Asset Bundles!
 
diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/.gitignore b/acceptance/bundle/templates/default-sql/output/my_default_sql/.gitignore
new file mode 100644
index 000000000..de811f118
--- /dev/null
+++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/.gitignore
@@ -0,0 +1,2 @@
+
+.databricks
diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/.vscode/extensions.json b/acceptance/bundle/templates/default-sql/output/my_default_sql/.vscode/extensions.json
new file mode 100644
index 000000000..8e1023465
--- /dev/null
+++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/.vscode/extensions.json
@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "databricks.databricks",
+        "redhat.vscode-yaml",
+        "databricks.sqltools-databricks-driver",
+    ]
+}
diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/.vscode/settings.json b/acceptance/bundle/templates/default-sql/output/my_default_sql/.vscode/settings.json
new file mode 100644
index 000000000..c641abe39
--- /dev/null
+++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/.vscode/settings.json
@@ -0,0 +1,27 @@
+{
+    "python.analysis.stubPath": ".vscode",
+    "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\<codecell\\>|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])",
+    "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------",
+    "python.testing.pytestArgs": [
+        "."
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true,
+    "python.analysis.extraPaths": ["src"],
+    "files.exclude": {
+        "**/*.egg-info": true,
+        "**/__pycache__": true,
+        ".pytest_cache": true,
+    },
+    "sqltools.connections": [
+        {
+            "connectionMethod": "VS Code Extension (beta)",
+            "catalog": "main",
+            "previewLimit": 50,
+            "driver": "Databricks",
+            "name": "databricks",
+            "path": "/sql/2.0/warehouses/f00dcafe"
+        }
+    ],
+    "sqltools.autoConnectTo": "",
+}
diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/README.md b/acceptance/bundle/templates/default-sql/output/my_default_sql/README.md
new file mode 100644
index 000000000..67ded153f
--- /dev/null
+++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/README.md
@@ -0,0 +1,41 @@
+# my_default_sql
+
+The 'my_default_sql' project was generated by using the default-sql template.
+
+## Getting started
+
+1. Install the Databricks CLI from https://docs.databricks.com/dev-tools/cli/install.html
+
+2. Authenticate to your Databricks workspace (if you have not done so already):
+    ```
+    $ databricks configure
+    ```
+
+3. To deploy a development copy of this project, type:
+    ```
+    $ databricks bundle deploy --target dev
+    ```
+    (Note that "dev" is the default target, so the `--target` parameter
+    is optional here.)
+
+    This deploys everything that's defined for this project.
+    For example, the default template would deploy a job called
+    `[dev yourname] my_default_sql_job` to your workspace.
+    You can find that job by opening your workpace and clicking on **Workflows**.
+
+4. Similarly, to deploy a production copy, type:
+   ```
+   $ databricks bundle deploy --target prod
+   ```
+
+5. To run a job, use the "run" command:
+   ```
+   $ databricks bundle run
+   ```
+
+6. Optionally, install developer tools such as the Databricks extension for Visual Studio Code from
+   https://docs.databricks.com/dev-tools/vscode-ext.html.
+
+7. For documentation on the Databricks Asset Bundles format used
+   for this project, and for CI/CD configuration, see
+   https://docs.databricks.com/dev-tools/bundles/index.html.
diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml b/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml
new file mode 100644
index 000000000..ab857287e
--- /dev/null
+++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml
@@ -0,0 +1,48 @@
+# This is a Databricks asset bundle definition for my_default_sql.
+# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
+bundle:
+  name: my_default_sql
+  uuid: <UUID>
+
+include:
+  - resources/*.yml
+
+# Variable declarations. These variables are assigned in the dev/prod targets below.
+variables:
+  warehouse_id:
+    description: The warehouse to use
+  catalog:
+    description: The catalog to use
+  schema:
+    description: The schema to use
+
+targets:
+  dev:
+    # The default target uses 'mode: development' to create a development copy.
+    # - Deployed resources get prefixed with '[dev my_user_name]'
+    # - Any job schedules and triggers are paused by default.
+    # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html.
+    mode: development
+    default: true
+    workspace:
+      host: $DATABRICKS_URL
+    variables:
+      warehouse_id: f00dcafe
+      catalog: main
+      schema: ${workspace.current_user.short_name}
+
+  prod:
+    mode: production
+    workspace:
+      host: $DATABRICKS_URL
+      # We explicitly specify /Workspace/Users/$USERNAME to make sure we only have a single copy.
+      root_path: /Workspace/Users/$USERNAME/.bundle/${bundle.name}/${bundle.target}
+    variables:
+      warehouse_id: f00dcafe
+      catalog: main
+      schema: default
+    permissions:
+      - user_name: $USERNAME
+        level: CAN_MANAGE
+    run_as:
+      user_name: $USERNAME
diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/resources/my_default_sql_sql.job.yml b/acceptance/bundle/templates/default-sql/output/my_default_sql/resources/my_default_sql_sql.job.yml
new file mode 100644
index 000000000..86de0f9db
--- /dev/null
+++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/resources/my_default_sql_sql.job.yml
@@ -0,0 +1,38 @@
+# A job running SQL queries on a SQL warehouse
+resources:
+  jobs:
+    my_default_sql_sql_job:
+      name: my_default_sql_sql_job
+
+      trigger:
+        # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger
+        periodic:
+          interval: 1
+          unit: DAYS
+
+      email_notifications:
+        on_failure:
+          - $USERNAME
+
+      parameters:
+        - name: catalog
+          default: ${var.catalog}
+        - name: schema
+          default: ${var.schema}
+        - name: bundle_target
+          default: ${bundle.target}
+
+      tasks:
+        - task_key: orders_raw
+          sql_task:
+            warehouse_id: ${var.warehouse_id}
+            file:
+              path: ../src/orders_raw.sql
+
+        - task_key: orders_daily
+          depends_on:
+            - task_key: orders_raw
+          sql_task:
+            warehouse_id: ${var.warehouse_id}
+            file:
+              path: ../src/orders_daily.sql
diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/README.md b/acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/README.md
new file mode 100644
index 000000000..5350d09cf
--- /dev/null
+++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/README.md
@@ -0,0 +1,4 @@
+# scratch
+
+This folder is reserved for personal, exploratory notebooks and SQL files.
+By default these are not committed to Git, as 'scratch' is listed in .gitignore.
diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/exploration.ipynb b/acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/exploration.ipynb
new file mode 100644
index 000000000..c3fd072e5
--- /dev/null
+++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/exploration.ipynb
@@ -0,0 +1,35 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "<UUID>",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "%sql\n",
+    "SELECT * FROM json.`/databricks-datasets/nyctaxi/sample/json/`"
+   ]
+  }
+ ],
+ "metadata": {
+  "application/vnd.databricks.v1+notebook": {
+   "dashboards": [],
+   "language": "python",
+   "notebookMetadata": {
+    "pythonIndentUnit": 2
+   },
+   "notebookName": "exploration",
+   "widgets": {}
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/src/orders_daily.sql b/acceptance/bundle/templates/default-sql/output/my_default_sql/src/orders_daily.sql
new file mode 100644
index 000000000..ea7b80b54
--- /dev/null
+++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/src/orders_daily.sql
@@ -0,0 +1,21 @@
+-- This query is executed using Databricks Workflows (see resources/my_default_sql_sql.job.yml)
+
+USE CATALOG {{catalog}};
+USE IDENTIFIER({{schema}});
+
+CREATE OR REPLACE MATERIALIZED VIEW
+  orders_daily
+AS SELECT
+  order_date, count(*) AS number_of_orders
+FROM
+  orders_raw
+
+WHERE if(
+  {{bundle_target}} = "prod",
+  true,
+
+  -- During development, only process a smaller range of data
+  order_date >= '2019-08-01' AND order_date < '2019-09-01'
+)
+
+GROUP BY order_date
diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/src/orders_raw.sql b/acceptance/bundle/templates/default-sql/output/my_default_sql/src/orders_raw.sql
new file mode 100644
index 000000000..79b1354cf
--- /dev/null
+++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/src/orders_raw.sql
@@ -0,0 +1,19 @@
+-- This query is executed using Databricks Workflows (see resources/my_default_sql_sql.job.yml)
+--
+-- The streaming table below ingests all JSON files in /databricks-datasets/retail-org/sales_orders/
+-- See also https://docs.databricks.com/sql/language-manual/sql-ref-syntax-ddl-create-streaming-table.html
+
+USE CATALOG {{catalog}};
+USE IDENTIFIER({{schema}});
+
+CREATE OR REFRESH STREAMING TABLE
+  orders_raw
+AS SELECT
+  customer_name,
+  DATE(TIMESTAMP(FROM_UNIXTIME(TRY_CAST(order_datetime AS BIGINT)))) AS order_date,
+  order_number
+FROM STREAM READ_FILES(
+  "/databricks-datasets/retail-org/sales_orders/",
+  format => "json",
+  header => true
+)
diff --git a/acceptance/bundle/templates/default-sql/script b/acceptance/bundle/templates/default-sql/script
new file mode 100644
index 000000000..66e7a14a2
--- /dev/null
+++ b/acceptance/bundle/templates/default-sql/script
@@ -0,0 +1,5 @@
+trace $CLI bundle init default-sql --config-file ./input.json --output-dir output
+
+cd output/my_default_sql
+trace $CLI bundle validate -t dev
+trace $CLI bundle validate -t prod
diff --git a/libs/testdiff/testdiff.go b/libs/testdiff/testdiff.go
index fef1d5ae2..232f38b41 100644
--- a/libs/testdiff/testdiff.go
+++ b/libs/testdiff/testdiff.go
@@ -17,18 +17,20 @@ func UnifiedDiff(filename1, filename2, s1, s2 string) string {
 	return fmt.Sprint(gotextdiff.ToUnified(filename1, filename2, s1, edits))
 }
 
-func AssertEqualTexts(t testutil.TestingT, filename1, filename2, expected, out string) {
+func AssertEqualTexts(t testutil.TestingT, filename1, filename2, expected, out string) bool {
 	t.Helper()
 	if len(out) < 1000 && len(expected) < 1000 {
 		// This shows full strings + diff which could be useful when debugging newlines
-		assert.Equal(t, expected, out, "%s vs %s", filename1, filename2)
+		return assert.Equal(t, expected, out, "%s vs %s", filename1, filename2)
 	} else {
 		// only show diff for large texts
 		diff := UnifiedDiff(filename1, filename2, expected, out)
 		if diff != "" {
 			t.Errorf("Diff:\n" + diff)
+			return false
 		}
 	}
+	return true
 }
 
 func AssertEqualJQ(t testutil.TestingT, expectedName, outName, expected, out string, ignorePaths []string) {

From 26f527ef64b552ad08940e63b92eef005021a804 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 20 Jan 2025 09:07:42 +0100
Subject: [PATCH 107/247] Fix incorrect TestingT.Errorf usage and enable
 linting for this (#2182)

## Changes
- Fix incorrect use Errorf on literal string. This resulted in garbage
output in tests diagnostics where % was replaced by "(MISSING)".
- Enable linter on testingT.Errorf.

Note, the autofix by the linter is wrong, it proposes `t.Errorf("%s",
string)` but it should be `t.Error(string)`. That can corrected manually
though.

## Tests
Linter was tested manually by reverting the fix on Errorf.
---
 .golangci.yaml            | 7 +++++++
 libs/testdiff/testdiff.go | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/.golangci.yaml b/.golangci.yaml
index ea6d65db1..8a83135ee 100644
--- a/.golangci.yaml
+++ b/.golangci.yaml
@@ -22,6 +22,13 @@ linters-settings:
     disable:
       - fieldalignment
       - shadow
+    settings:
+      printf:
+        funcs:
+          - (github.com/databricks/cli/internal/testutil.TestingT).Infof
+          - (github.com/databricks/cli/internal/testutil.TestingT).Errorf
+          - (github.com/databricks/cli/internal/testutil.TestingT).Fatalf
+          - (github.com/databricks/cli/internal/testutil.TestingT).Skipf
   gofmt:
     rewrite-rules:
       - pattern: 'a[b:len(a)]'
diff --git a/libs/testdiff/testdiff.go b/libs/testdiff/testdiff.go
index 232f38b41..f65adf7f7 100644
--- a/libs/testdiff/testdiff.go
+++ b/libs/testdiff/testdiff.go
@@ -26,7 +26,7 @@ func AssertEqualTexts(t testutil.TestingT, filename1, filename2, expected, out s
 		// only show diff for large texts
 		diff := UnifiedDiff(filename1, filename2, expected, out)
 		if diff != "" {
-			t.Errorf("Diff:\n" + diff)
+			t.Error("Diff:\n" + diff)
 			return false
 		}
 	}

From 64fc1c8fe74071defd1b16d8cffd89d6c53def14 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 20 Jan 2025 09:57:48 +0100
Subject: [PATCH 108/247] Add NoLog option on testcli.Runner (#2183)

## Changes
Setting Verbose=false on testcli.Runner disables all logging related to
running process (stdout, stderr, error, args).

I'm using this in #2184 where I'm using testcli runner to run acceptance
tests and seeing all output is not useful.

## Tests
Manually inspecting test output in #2184
---
 internal/testcli/runner.go | 35 ++++++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/internal/testcli/runner.go b/internal/testcli/runner.go
index d32fa3947..f462f44fc 100644
--- a/internal/testcli/runner.go
+++ b/internal/testcli/runner.go
@@ -39,6 +39,8 @@ type Runner struct {
 	StderrLines <-chan string
 
 	errch <-chan error
+
+	Verbose bool
 }
 
 func consumeLines(ctx context.Context, wg *sync.WaitGroup, r io.Reader) <-chan string {
@@ -139,7 +141,9 @@ func (r *Runner) RunBackground() {
 	go func() {
 		err := root.Execute(ctx, cli)
 		if err != nil {
-			r.Logf("Error running command: %s", err)
+			if r.Verbose {
+				r.Logf("Error running command: %s", err)
+			}
 		}
 
 		// Close pipes to signal EOF.
@@ -154,7 +158,9 @@ func (r *Runner) RunBackground() {
 			// Make a copy of the buffer such that it remains "unread".
 			scanner := bufio.NewScanner(bytes.NewBuffer(r.stdout.Bytes()))
 			for scanner.Scan() {
-				r.Logf("[databricks stdout]: %s", scanner.Text())
+				if r.Verbose {
+					r.Logf("[databricks stdout]: %s", scanner.Text())
+				}
 			}
 		}
 
@@ -162,7 +168,9 @@ func (r *Runner) RunBackground() {
 			// Make a copy of the buffer such that it remains "unread".
 			scanner := bufio.NewScanner(bytes.NewBuffer(r.stderr.Bytes()))
 			for scanner.Scan() {
-				r.Logf("[databricks stderr]: %s", scanner.Text())
+				if r.Verbose {
+					r.Logf("[databricks stderr]: %s", scanner.Text())
+				}
 			}
 		}
 
@@ -196,18 +204,24 @@ func (r *Runner) Run() (bytes.Buffer, bytes.Buffer, error) {
 	cli.SetErr(&stderr)
 	cli.SetArgs(r.args)
 
-	r.Logf("  args: %s", strings.Join(r.args, ", "))
+	if r.Verbose {
+		r.Logf("  args: %s", strings.Join(r.args, ", "))
+	}
 
 	err := root.Execute(ctx, cli)
 	if err != nil {
-		r.Logf(" error: %s", err)
+		if r.Verbose {
+			r.Logf(" error: %s", err)
+		}
 	}
 
 	if stdout.Len() > 0 {
 		// Make a copy of the buffer such that it remains "unread".
 		scanner := bufio.NewScanner(bytes.NewBuffer(stdout.Bytes()))
 		for scanner.Scan() {
-			r.Logf("stdout: %s", scanner.Text())
+			if r.Verbose {
+				r.Logf("stdout: %s", scanner.Text())
+			}
 		}
 	}
 
@@ -215,7 +229,9 @@ func (r *Runner) Run() (bytes.Buffer, bytes.Buffer, error) {
 		// Make a copy of the buffer such that it remains "unread".
 		scanner := bufio.NewScanner(bytes.NewBuffer(stderr.Bytes()))
 		for scanner.Scan() {
-			r.Logf("stderr: %s", scanner.Text())
+			if r.Verbose {
+				r.Logf("stderr: %s", scanner.Text())
+			}
 		}
 	}
 
@@ -275,8 +291,9 @@ func NewRunner(t testutil.TestingT, ctx context.Context, args ...string) *Runner
 	return &Runner{
 		TestingT: t,
 
-		ctx:  ctx,
-		args: args,
+		ctx:     ctx,
+		args:    args,
+		Verbose: true,
 	}
 }
 

From 7034793d1d64ce669dc2a10c9ae2ddb8a212defb Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 20 Jan 2025 10:26:29 +0100
Subject: [PATCH 109/247] Run 'ruff format' in quiet mode (#2187)

Otherwise it prints "83 files left unchanged".
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 1ab247406..4b66c9254 100644
--- a/Makefile
+++ b/Makefile
@@ -14,7 +14,7 @@ lintcheck:
 # formatting/goimports will not be applied by 'make lint'. However, it will be applied by 'make fmt'.
 # If you need to ensure that formatting & imports are always fixed, do "make fmt lint"
 fmt:
-	ruff format
+	ruff format -q
 	golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix ./...
 
 test:

From 31c10c1b82e45f5b10e07df03e013a75c884b703 Mon Sep 17 00:00:00 2001
From: Gleb Kanterov <kanterov@users.noreply.github.com>
Date: Mon, 20 Jan 2025 11:15:11 +0100
Subject: [PATCH 110/247] Add experimental-jobs-as-code template (#2177)

## Changes

Add experimental-jobs-as-code template allowing defining jobs using
Python instead of YAML through the `databricks-bundles` PyPI package.

## Tests

Manually and acceptance tests.
---
 acceptance/acceptance_test.go                 |  23 ++++
 .../experimental-jobs-as-code/input.json      |   5 +
 .../experimental-jobs-as-code/output.txt      |  85 ++++++++++++++
 .../output/my_jobs_as_code/.gitignore         |   8 ++
 .../output/my_jobs_as_code/README.md          |  58 ++++++++++
 .../output/my_jobs_as_code/databricks.yml     |  48 ++++++++
 .../output/my_jobs_as_code/fixtures/.gitkeep  |  22 ++++
 .../output/my_jobs_as_code/pyproject.toml     |  49 ++++++++
 .../my_jobs_as_code/resources/__init__.py     |  16 +++
 .../resources/my_jobs_as_code_job.py          |  67 +++++++++++
 .../output/my_jobs_as_code/scratch/README.md  |   4 +
 .../output/my_jobs_as_code/setup.py           |  18 +++
 .../src/my_jobs_as_code/__init__.py           |   0
 .../src/my_jobs_as_code/main.py               |  25 ++++
 .../output/my_jobs_as_code/src/notebook.ipynb |  75 ++++++++++++
 .../output/my_jobs_as_code/tests/main_test.py |   8 ++
 .../experimental-jobs-as-code/script          |  12 ++
 cmd/bundle/init.go                            |   5 +
 .../databricks_template_schema.json           |  28 +++++
 .../library/versions.tmpl                     |   7 ++
 .../template/__preamble.tmpl                  |  30 +++++
 .../template/{{.project_name}}/.gitignore     |   8 ++
 .../template/{{.project_name}}/README.md.tmpl |  60 ++++++++++
 .../{{.project_name}}/databricks.yml.tmpl     |  51 +++++++++
 .../{{.project_name}}/fixtures/.gitkeep.tmpl  |  27 +++++
 .../{{.project_name}}/pyproject.toml.tmpl     |  57 +++++++++
 .../{{.project_name}}/resources/__init__.py   |  16 +++
 .../resources/{{.project_name}}_job.py.tmpl   | 108 ++++++++++++++++++
 .../{{.project_name}}_pipeline.py.tmpl        |  24 ++++
 .../{{.project_name}}/scratch/README.md       |   4 +
 .../template/{{.project_name}}/setup.py.tmpl  |  18 +++
 .../src/dlt_pipeline.ipynb.tmpl               | 104 +++++++++++++++++
 .../{{.project_name}}/src/notebook.ipynb.tmpl |  79 +++++++++++++
 .../src/{{.project_name}}/__init__.py.tmpl    |   0
 .../src/{{.project_name}}/main.py.tmpl        |  25 ++++
 .../{{.project_name}}/tests/main_test.py.tmpl |   8 ++
 36 files changed, 1182 insertions(+)
 create mode 100644 acceptance/bundle/templates/experimental-jobs-as-code/input.json
 create mode 100644 acceptance/bundle/templates/experimental-jobs-as-code/output.txt
 create mode 100644 acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/.gitignore
 create mode 100644 acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/README.md
 create mode 100644 acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml
 create mode 100644 acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/fixtures/.gitkeep
 create mode 100644 acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/pyproject.toml
 create mode 100644 acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/__init__.py
 create mode 100644 acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py
 create mode 100644 acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/scratch/README.md
 create mode 100644 acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/setup.py
 create mode 100644 acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/my_jobs_as_code/__init__.py
 create mode 100644 acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/my_jobs_as_code/main.py
 create mode 100644 acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/notebook.ipynb
 create mode 100644 acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/tests/main_test.py
 create mode 100644 acceptance/bundle/templates/experimental-jobs-as-code/script
 create mode 100644 libs/template/templates/experimental-jobs-as-code/databricks_template_schema.json
 create mode 100644 libs/template/templates/experimental-jobs-as-code/library/versions.tmpl
 create mode 100644 libs/template/templates/experimental-jobs-as-code/template/__preamble.tmpl
 create mode 100644 libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/.gitignore
 create mode 100644 libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/README.md.tmpl
 create mode 100644 libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/databricks.yml.tmpl
 create mode 100644 libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/fixtures/.gitkeep.tmpl
 create mode 100644 libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/pyproject.toml.tmpl
 create mode 100644 libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/__init__.py
 create mode 100644 libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/{{.project_name}}_job.py.tmpl
 create mode 100644 libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/{{.project_name}}_pipeline.py.tmpl
 create mode 100644 libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/scratch/README.md
 create mode 100644 libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/setup.py.tmpl
 create mode 100644 libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl
 create mode 100644 libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/src/notebook.ipynb.tmpl
 create mode 100644 libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/src/{{.project_name}}/__init__.py.tmpl
 create mode 100644 libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/src/{{.project_name}}/main.py.tmpl
 create mode 100644 libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/tests/main_test.py.tmpl

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 2dfe2bce4..5f1181313 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -8,6 +8,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"slices"
 	"sort"
@@ -393,6 +394,16 @@ func CopyDir(src, dst string, inputs, outputs map[string]bool) error {
 }
 
 func ListDir(t *testing.T, src string) ([]string, error) {
+	// exclude folders in .gitignore from comparison
+	ignored := []string{
+		"\\.ruff_cache",
+		"\\.venv",
+		".*\\.egg-info",
+		"__pycache__",
+		// depends on uv version
+		"uv.lock",
+	}
+
 	var files []string
 	err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -400,7 +411,19 @@ func ListDir(t *testing.T, src string) ([]string, error) {
 		}
 
 		if info.IsDir() {
+			for _, ignoredFolder := range ignored {
+				if matched, _ := regexp.MatchString(ignoredFolder, info.Name()); matched {
+					return filepath.SkipDir
+				}
+			}
+
 			return nil
+		} else {
+			for _, ignoredFolder := range ignored {
+				if matched, _ := regexp.MatchString(ignoredFolder, info.Name()); matched {
+					return nil
+				}
+			}
 		}
 
 		relPath, err := filepath.Rel(src, path)
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/input.json b/acceptance/bundle/templates/experimental-jobs-as-code/input.json
new file mode 100644
index 000000000..748076c75
--- /dev/null
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/input.json
@@ -0,0 +1,5 @@
+{
+  "project_name": "my_jobs_as_code",
+  "include_notebook": "yes",
+  "include_python": "yes"
+}
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output.txt b/acceptance/bundle/templates/experimental-jobs-as-code/output.txt
new file mode 100644
index 000000000..1aa8a94d5
--- /dev/null
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output.txt
@@ -0,0 +1,85 @@
+
+>>> $CLI bundle init experimental-jobs-as-code --config-file ./input.json --output-dir output
+
+Welcome to (EXPERIMENTAL) "Jobs as code" template for Databricks Asset Bundles!
+Workspace to use (auto-detected, edit in 'my_jobs_as_code/databricks.yml'): $DATABRICKS_URL
+
+✨ Your new project has been created in the 'my_jobs_as_code' directory!
+
+Please refer to the README.md file for "getting started" instructions.
+See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.
+
+>>> $CLI bundle validate -t dev --output json
+{
+  "jobs": {
+    "my_jobs_as_code_job": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/my_jobs_as_code/dev/state/metadata.json"
+      },
+      "edit_mode": "UI_LOCKED",
+      "email_notifications": {
+        "on_failure": [
+          "$USERNAME"
+        ]
+      },
+      "format": "MULTI_TASK",
+      "job_clusters": [
+        {
+          "job_cluster_key": "job_cluster",
+          "new_cluster": {
+            "autoscale": {
+              "max_workers": 4,
+              "min_workers": 1
+            },
+            "node_type_id": "i3.xlarge",
+            "spark_version": "15.4.x-scala2.12"
+          }
+        }
+      ],
+      "max_concurrent_runs": 4,
+      "name": "[dev $USERNAME] my_jobs_as_code_job",
+      "permissions": [],
+      "queue": {
+        "enabled": true
+      },
+      "tags": {
+        "dev": "$USERNAME"
+      },
+      "tasks": [
+        {
+          "job_cluster_key": "job_cluster",
+          "notebook_task": {
+            "notebook_path": "/Workspace/Users/$USERNAME/.bundle/my_jobs_as_code/dev/files/src/notebook"
+          },
+          "task_key": "notebook_task"
+        },
+        {
+          "depends_on": [
+            {
+              "task_key": "notebook_task"
+            }
+          ],
+          "job_cluster_key": "job_cluster",
+          "libraries": [
+            {
+              "whl": "dist/*.whl"
+            }
+          ],
+          "python_wheel_task": {
+            "entry_point": "main",
+            "package_name": "my_jobs_as_code"
+          },
+          "task_key": "main_task"
+        }
+      ],
+      "trigger": {
+        "pause_status": "PAUSED",
+        "periodic": {
+          "interval": 1,
+          "unit": "DAYS"
+        }
+      }
+    }
+  }
+}
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/.gitignore b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/.gitignore
new file mode 100644
index 000000000..0dab7f499
--- /dev/null
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/.gitignore
@@ -0,0 +1,8 @@
+.databricks/
+build/
+dist/
+__pycache__/
+*.egg-info
+.venv/
+scratch/**
+!scratch/README.md
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/README.md b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/README.md
new file mode 100644
index 000000000..8c429c6e5
--- /dev/null
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/README.md
@@ -0,0 +1,58 @@
+# my_jobs_as_code
+
+The 'my_jobs_as_code' project was generated by using the "Jobs as code" template.
+
+## Prerequisites
+
+1. Install Databricks CLI 0.238 or later.
+   See [Install or update the Databricks CLI](https://docs.databricks.com/en/dev-tools/cli/install.html).
+
+2. Install uv. See [Installing uv](https://docs.astral.sh/uv/getting-started/installation/).
+   We use uv to create a virtual environment and install the required dependencies.
+
+3. Authenticate to your Databricks workspace if you have not done so already:
+    ```
+    $ databricks configure
+    ```
+
+4. Optionally, install developer tools such as the Databricks extension for Visual Studio Code from
+   https://docs.databricks.com/dev-tools/vscode-ext.html. Or read the "getting started" documentation for
+   **Databricks Connect** for instructions on running the included Python code from a different IDE.
+
+5. For documentation on the Databricks Asset Bundles format used
+   for this project, and for CI/CD configuration, see
+   https://docs.databricks.com/dev-tools/bundles/index.html.
+
+## Deploy and run jobs
+
+1. Create a new virtual environment and install the required dependencies:
+    ```
+    $ uv sync
+    ```
+
+2. To deploy the bundle to the development target:
+    ```
+    $ databricks bundle deploy --target dev
+    ```
+
+   *(Note that "dev" is the default target, so the `--target` parameter is optional here.)*
+
+   This deploys everything that's defined for this project.
+   For example, the default template would deploy a job called
+   `[dev yourname] my_jobs_as_code_job` to your workspace.
+   You can find that job by opening your workspace and clicking on **Workflows**.
+
+3. Similarly, to deploy a production copy, type:
+   ```
+   $ databricks bundle deploy --target prod
+   ```
+
+   Note that the default job from the template has a schedule that runs every day
+   (defined in resources/my_jobs_as_code_job.py). The schedule
+   is paused when deploying in development mode (see [Databricks Asset Bundle deployment modes](
+   https://docs.databricks.com/dev-tools/bundles/deployment-modes.html)).
+
+4. To run a job:
+   ```
+   $ databricks bundle run
+   ```
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml
new file mode 100644
index 000000000..fd87aa381
--- /dev/null
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml
@@ -0,0 +1,48 @@
+# This is a Databricks asset bundle definition for my_jobs_as_code.
+# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
+bundle:
+  name: my_jobs_as_code
+  uuid: <UUID>
+
+experimental:
+  python:
+    # Activate virtual environment before loading resources defined in Python.
+    # If disabled, defaults to using the Python interpreter available in the current shell.
+    venv_path: .venv
+    # Functions called to load resources defined in Python. See resources/__init__.py
+    resources:
+      - "resources:load_resources"
+
+artifacts:
+  default:
+    type: whl
+    path: .
+    # We use timestamp as Local version identifier (https://peps.python.org/pep-0440/#local-version-identifiers.)
+    # to ensure that changes to wheel package are picked up when used on all-purpose clusters
+    build: LOCAL_VERSION=$(date +%Y%m%d.%H%M%S) uv build
+
+include:
+  - resources/*.yml
+
+targets:
+  dev:
+    # The default target uses 'mode: development' to create a development copy.
+    # - Deployed resources get prefixed with '[dev my_user_name]'
+    # - Any job schedules and triggers are paused by default.
+    # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html.
+    mode: development
+    default: true
+    workspace:
+      host: $DATABRICKS_URL
+
+  prod:
+    mode: production
+    workspace:
+      host: $DATABRICKS_URL
+      # We explicitly specify /Workspace/Users/$USERNAME to make sure we only have a single copy.
+      root_path: /Workspace/Users/$USERNAME/.bundle/${bundle.name}/${bundle.target}
+    permissions:
+      - user_name: $USERNAME
+        level: CAN_MANAGE
+    run_as:
+      user_name: $USERNAME
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/fixtures/.gitkeep b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/fixtures/.gitkeep
new file mode 100644
index 000000000..fa25d2745
--- /dev/null
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/fixtures/.gitkeep
@@ -0,0 +1,22 @@
+# Fixtures
+
+This folder is reserved for fixtures, such as CSV files.
+
+Below is an example of how to load fixtures as a data frame:
+
+```
+import pandas as pd
+import os
+
+def get_absolute_path(*relative_parts):
+    if 'dbutils' in globals():
+        base_dir = os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) # type: ignore
+        path = os.path.normpath(os.path.join(base_dir, *relative_parts))
+        return path if path.startswith("/Workspace") else "/Workspace" + path
+    else:
+        return os.path.join(*relative_parts)
+
+csv_file = get_absolute_path("..", "fixtures", "mycsv.csv")
+df = pd.read_csv(csv_file)
+display(df)
+```
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/pyproject.toml b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/pyproject.toml
new file mode 100644
index 000000000..28240e3ec
--- /dev/null
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/pyproject.toml
@@ -0,0 +1,49 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "my_jobs_as_code"
+requires-python = ">=3.10"
+description = "wheel file based on my_jobs_as_code"
+
+# Dependencies in case the output wheel file is used as a library dependency.
+# For defining dependencies, when this package is used in Databricks, see:
+# https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
+#
+# Example:
+# dependencies = [
+#     "requests==x.y.z",
+# ]
+dependencies = [
+]
+
+# see setup.py
+dynamic = ["version"]
+
+[project.entry-points.packages]
+main = "my_jobs_as_code.main:main"
+
+[tool.setuptools]
+py-modules = ["resources", "my_jobs_as_code"]
+
+[tool.uv]
+## Dependencies for local development
+dev-dependencies = [
+    "databricks-bundles==0.7.0",
+
+    ## Add code completion support for DLT
+    # "databricks-dlt",
+
+    ## databricks-connect can be used to run parts of this project locally.
+    ## See https://docs.databricks.com/dev-tools/databricks-connect.html.
+    ##
+    ## Uncomment line below to install a version of db-connect that corresponds to
+    ## the Databricks Runtime version used for this project.
+    # "databricks-connect>=15.4,<15.5",
+]
+
+override-dependencies = [
+    # pyspark package conflicts with 'databricks-connect'
+    "pyspark; sys_platform == 'never'",
+]
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/__init__.py b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/__init__.py
new file mode 100644
index 000000000..fbcb9dc5f
--- /dev/null
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/__init__.py
@@ -0,0 +1,16 @@
+from databricks.bundles.core import (
+    Bundle,
+    Resources,
+    load_resources_from_current_package_module,
+)
+
+
+def load_resources(bundle: Bundle) -> Resources:
+    """
+    'load_resources' function is referenced in databricks.yml and is responsible for loading
+    bundle resources defined in Python code. This function is called by Databricks CLI during
+    bundle deployment. After deployment, this function is not used.
+    """
+
+    # the default implementation loads all Python files in 'resources' directory
+    return load_resources_from_current_package_module()
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py
new file mode 100644
index 000000000..4854d656f
--- /dev/null
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py
@@ -0,0 +1,67 @@
+from databricks.bundles.jobs import Job
+
+"""
+The main job for my_jobs_as_code.
+"""
+
+
+my_jobs_as_code_job = Job.from_dict(
+    {
+        "name": "my_jobs_as_code_job",
+        "trigger": {
+            # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger
+            "periodic": {
+                "interval": 1,
+                "unit": "DAYS",
+            },
+        },
+        "email_notifications": {
+            "on_failure": [
+                "$USERNAME",
+            ],
+        },
+        "tasks": [
+            {
+                "task_key": "notebook_task",
+                "job_cluster_key": "job_cluster",
+                "notebook_task": {
+                    "notebook_path": "src/notebook.ipynb",
+                },
+            },
+            {
+                "task_key": "main_task",
+                "depends_on": [
+                    {
+                        "task_key": "notebook_task",
+                    },
+                ],
+                "job_cluster_key": "job_cluster",
+                "python_wheel_task": {
+                    "package_name": "my_jobs_as_code",
+                    "entry_point": "main",
+                },
+                "libraries": [
+                    # By default we just include the .whl file generated for the my_jobs_as_code package.
+                    # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
+                    # for more information on how to add other libraries.
+                    {
+                        "whl": "dist/*.whl",
+                    },
+                ],
+            },
+        ],
+        "job_clusters": [
+            {
+                "job_cluster_key": "job_cluster",
+                "new_cluster": {
+                    "spark_version": "15.4.x-scala2.12",
+                    "node_type_id": "i3.xlarge",
+                    "autoscale": {
+                        "min_workers": 1,
+                        "max_workers": 4,
+                    },
+                },
+            },
+        ],
+    }
+)
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/scratch/README.md b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/scratch/README.md
new file mode 100644
index 000000000..e6cfb81b4
--- /dev/null
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/scratch/README.md
@@ -0,0 +1,4 @@
+# scratch
+
+This folder is reserved for personal, exploratory notebooks.
+By default these are not committed to Git, as 'scratch' is listed in .gitignore.
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/setup.py b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/setup.py
new file mode 100644
index 000000000..ba284ba82
--- /dev/null
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/setup.py
@@ -0,0 +1,18 @@
+"""
+setup.py configuration script describing how to build and package this project.
+
+This file is primarily used by the setuptools library and typically should not
+be executed directly. See README.md for how to deploy, test, and run
+the my_jobs_as_code project.
+"""
+
+import os
+
+from setuptools import setup
+
+local_version = os.getenv("LOCAL_VERSION")
+version = "0.0.1"
+
+setup(
+    version=f"{version}+{local_version}" if local_version else version,
+)
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/my_jobs_as_code/__init__.py b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/my_jobs_as_code/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/my_jobs_as_code/main.py b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/my_jobs_as_code/main.py
new file mode 100644
index 000000000..5ae344c7e
--- /dev/null
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/my_jobs_as_code/main.py
@@ -0,0 +1,25 @@
+from pyspark.sql import SparkSession, DataFrame
+
+
+def get_taxis(spark: SparkSession) -> DataFrame:
+    return spark.read.table("samples.nyctaxi.trips")
+
+
+# Create a new Databricks Connect session. If this fails,
+# check that you have configured Databricks Connect correctly.
+# See https://docs.databricks.com/dev-tools/databricks-connect.html.
+def get_spark() -> SparkSession:
+    try:
+        from databricks.connect import DatabricksSession
+
+        return DatabricksSession.builder.getOrCreate()
+    except ImportError:
+        return SparkSession.builder.getOrCreate()
+
+
+def main():
+    get_taxis(get_spark()).show(5)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/notebook.ipynb b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/notebook.ipynb
new file mode 100644
index 000000000..9bc3f1560
--- /dev/null
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/notebook.ipynb
@@ -0,0 +1,75 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "<UUID>",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "source": [
+    "# Default notebook\n",
+    "\n",
+    "This default notebook is executed using Databricks Workflows as defined in resources/my_jobs_as_code.job.yml."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "<UUID>",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from my_jobs_as_code import main\n",
+    "\n",
+    "main.get_taxis(spark).show(10)"
+   ]
+  }
+ ],
+ "metadata": {
+  "application/vnd.databricks.v1+notebook": {
+   "dashboards": [],
+   "language": "python",
+   "notebookMetadata": {
+    "pythonIndentUnit": 2
+   },
+   "notebookName": "notebook",
+   "widgets": {}
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/tests/main_test.py b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/tests/main_test.py
new file mode 100644
index 000000000..13e100ee2
--- /dev/null
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/tests/main_test.py
@@ -0,0 +1,8 @@
+from my_jobs_as_code.main import get_taxis, get_spark
+
+# running tests requires installing databricks-connect, e.g. by uncommenting it in pyproject.toml
+
+
+def test_main():
+    taxis = get_taxis(get_spark())
+    assert taxis.count() > 5
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/script b/acceptance/bundle/templates/experimental-jobs-as-code/script
new file mode 100644
index 000000000..2209aa7ab
--- /dev/null
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/script
@@ -0,0 +1,12 @@
+trace $CLI bundle init experimental-jobs-as-code --config-file ./input.json --output-dir output
+
+cd output/my_jobs_as_code
+
+# silence uv output because it's non-deterministic
+uv sync 2> /dev/null
+
+# remove version constraint because it always creates a warning on dev builds
+cat databricks.yml | grep -v databricks_cli_version > databricks.yml.new
+mv databricks.yml.new databricks.yml
+
+trace $CLI bundle validate -t dev --output json | jq ".resources"
diff --git a/cmd/bundle/init.go b/cmd/bundle/init.go
index 687c141ec..6b93fd1e5 100644
--- a/cmd/bundle/init.go
+++ b/cmd/bundle/init.go
@@ -59,6 +59,11 @@ var nativeTemplates = []nativeTemplate{
 		hidden:      true,
 		description: "The default PyDABs template",
 	},
+	{
+		name:        "experimental-jobs-as-code",
+		hidden:      true,
+		description: "Jobs as code template (experimental)",
+	},
 	{
 		name:        customTemplate,
 		description: "Bring your own template",
diff --git a/libs/template/templates/experimental-jobs-as-code/databricks_template_schema.json b/libs/template/templates/experimental-jobs-as-code/databricks_template_schema.json
new file mode 100644
index 000000000..00d59af5f
--- /dev/null
+++ b/libs/template/templates/experimental-jobs-as-code/databricks_template_schema.json
@@ -0,0 +1,28 @@
+{
+    "welcome_message": "\nWelcome to (EXPERIMENTAL) \"Jobs as code\" template for Databricks Asset Bundles!",
+    "properties": {
+        "project_name": {
+            "type": "string",
+            "default": "jobs_as_code_project",
+            "description": "Please provide the following details to tailor the template to your preferences.\n\nUnique name for this project",
+            "order": 1,
+            "pattern": "^[A-Za-z0-9_]+$",
+            "pattern_match_failure_message": "Name must consist of letters, numbers, and underscores."
+        },
+        "include_notebook": {
+            "type": "string",
+            "default": "yes",
+            "enum": ["yes", "no"],
+            "description": "Include a stub (sample) notebook in '{{.project_name}}{{path_separator}}src'",
+            "order": 2
+        },
+        "include_python": {
+            "type": "string",
+            "default": "yes",
+            "enum": ["yes", "no"],
+            "description": "Include a stub (sample) Python package in '{{.project_name}}/src'",
+            "order": 3
+        }
+    },
+    "success_message": "Workspace to use (auto-detected, edit in '{{.project_name}}/databricks.yml'): {{workspace_host}}\n\n✨ Your new project has been created in the '{{.project_name}}' directory!\n\nPlease refer to the README.md file for \"getting started\" instructions.\nSee also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html."
+}
diff --git a/libs/template/templates/experimental-jobs-as-code/library/versions.tmpl b/libs/template/templates/experimental-jobs-as-code/library/versions.tmpl
new file mode 100644
index 000000000..7d0c88e7d
--- /dev/null
+++ b/libs/template/templates/experimental-jobs-as-code/library/versions.tmpl
@@ -0,0 +1,7 @@
+{{define "latest_lts_dbr_version" -}}
+  15.4.x-scala2.12
+{{- end}}
+
+{{define "latest_lts_db_connect_version_spec" -}}
+  >=15.4,<15.5
+{{- end}}
diff --git a/libs/template/templates/experimental-jobs-as-code/template/__preamble.tmpl b/libs/template/templates/experimental-jobs-as-code/template/__preamble.tmpl
new file mode 100644
index 000000000..2f8e8ae3e
--- /dev/null
+++ b/libs/template/templates/experimental-jobs-as-code/template/__preamble.tmpl
@@ -0,0 +1,30 @@
+# Preamble
+
+This file only template directives; it is skipped for the actual output.
+
+{{skip "__preamble"}}
+
+# TODO add DLT support, placeholder for now
+{{$notDLT := true }}
+{{$notNotebook := not (eq .include_notebook "yes")}}
+{{$notPython := not (eq .include_python "yes")}}
+
+{{if $notPython}}
+  {{skip "{{.project_name}}/src/{{.project_name}}"}}
+  {{skip "{{.project_name}}/tests/main_test.py"}}
+{{end}}
+
+{{if $notDLT}}
+  {{skip "{{.project_name}}/src/dlt_pipeline.ipynb"}}
+  {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline.py"}}
+{{end}}
+
+{{if $notNotebook}}
+  {{skip "{{.project_name}}/src/notebook.ipynb"}}
+{{end}}
+
+{{if (and $notDLT $notNotebook $notPython)}}
+  {{skip "{{.project_name}}/resources/{{.project_name}}_job.py"}}
+{{else}}
+  {{skip "{{.project_name}}/resources/.gitkeep"}}
+{{end}}
diff --git a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/.gitignore b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/.gitignore
new file mode 100644
index 000000000..0dab7f499
--- /dev/null
+++ b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/.gitignore
@@ -0,0 +1,8 @@
+.databricks/
+build/
+dist/
+__pycache__/
+*.egg-info
+.venv/
+scratch/**
+!scratch/README.md
diff --git a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/README.md.tmpl b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/README.md.tmpl
new file mode 100644
index 000000000..497ce3723
--- /dev/null
+++ b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/README.md.tmpl
@@ -0,0 +1,60 @@
+# {{.project_name}}
+
+The '{{.project_name}}' project was generated by using the "Jobs as code" template.
+
+## Prerequisites
+
+1. Install Databricks CLI 0.238 or later.
+   See [Install or update the Databricks CLI](https://docs.databricks.com/en/dev-tools/cli/install.html).
+
+2. Install uv. See [Installing uv](https://docs.astral.sh/uv/getting-started/installation/).
+   We use uv to create a virtual environment and install the required dependencies.
+
+3. Authenticate to your Databricks workspace if you have not done so already:
+    ```
+    $ databricks configure
+    ```
+
+4. Optionally, install developer tools such as the Databricks extension for Visual Studio Code from
+   https://docs.databricks.com/dev-tools/vscode-ext.html.
+   {{- if (eq .include_python "yes") }} Or read the "getting started" documentation for
+   **Databricks Connect** for instructions on running the included Python code from a different IDE.
+   {{- end}}
+
+5. For documentation on the Databricks Asset Bundles format used
+   for this project, and for CI/CD configuration, see
+   https://docs.databricks.com/dev-tools/bundles/index.html.
+
+## Deploy and run jobs
+
+1. Create a new virtual environment and install the required dependencies:
+    ```
+    $ uv sync
+    ```
+
+2. To deploy the bundle to the development target:
+    ```
+    $ databricks bundle deploy --target dev
+    ```
+
+   *(Note that "dev" is the default target, so the `--target` parameter is optional here.)*
+
+   This deploys everything that's defined for this project.
+   For example, the default template would deploy a job called
+   `[dev yourname] {{.project_name}}_job` to your workspace.
+   You can find that job by opening your workspace and clicking on **Workflows**.
+
+3. Similarly, to deploy a production copy, type:
+   ```
+   $ databricks bundle deploy --target prod
+   ```
+
+   Note that the default job from the template has a schedule that runs every day
+   (defined in resources/{{.project_name}}_job.py). The schedule
+   is paused when deploying in development mode (see [Databricks Asset Bundle deployment modes](
+   https://docs.databricks.com/dev-tools/bundles/deployment-modes.html)).
+
+4. To run a job:
+   ```
+   $ databricks bundle run
+   ```
diff --git a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/databricks.yml.tmpl
new file mode 100644
index 000000000..758ec3f16
--- /dev/null
+++ b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/databricks.yml.tmpl
@@ -0,0 +1,51 @@
+# This is a Databricks asset bundle definition for {{.project_name}}.
+# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
+bundle:
+  name: {{.project_name}}
+  uuid: {{bundle_uuid}}
+  databricks_cli_version: ">= 0.238.0"
+
+experimental:
+  python:
+    # Activate virtual environment before loading resources defined in Python.
+    # If disabled, defaults to using the Python interpreter available in the current shell.
+    venv_path: .venv
+    # Functions called to load resources defined in Python. See resources/__init__.py
+    resources:
+      - "resources:load_resources"
+
+{{ if .include_python -}}
+artifacts:
+  default:
+    type: whl
+    path: .
+    # We use timestamp as Local version identifier (https://peps.python.org/pep-0440/#local-version-identifiers.)
+    # to ensure that changes to wheel package are picked up when used on all-purpose clusters
+    build: LOCAL_VERSION=$(date +%Y%m%d.%H%M%S) uv build
+
+{{ end -}}
+include:
+  - resources/*.yml
+
+targets:
+  dev:
+    # The default target uses 'mode: development' to create a development copy.
+    # - Deployed resources get prefixed with '[dev my_user_name]'
+    # - Any job schedules and triggers are paused by default.
+    # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html.
+    mode: development
+    default: true
+    workspace:
+      host: {{workspace_host}}
+
+  prod:
+    mode: production
+    workspace:
+      host: {{workspace_host}}
+      # We explicitly specify /Workspace/Users/{{user_name}} to make sure we only have a single copy.
+      root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target}
+    permissions:
+      - {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}}
+        level: CAN_MANAGE
+    run_as:
+      {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}}
diff --git a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/fixtures/.gitkeep.tmpl b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/fixtures/.gitkeep.tmpl
new file mode 100644
index 000000000..ee9570302
--- /dev/null
+++ b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/fixtures/.gitkeep.tmpl
@@ -0,0 +1,27 @@
+# Fixtures
+{{- /*
+We don't want to have too many README.md files, since they
+stand out so much. But we do need to have a file here to make
+sure the folder is added to Git.
+*/}}
+
+This folder is reserved for fixtures, such as CSV files.
+
+Below is an example of how to load fixtures as a data frame:
+
+```
+import pandas as pd
+import os
+
+def get_absolute_path(*relative_parts):
+    if 'dbutils' in globals():
+        base_dir = os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) # type: ignore
+        path = os.path.normpath(os.path.join(base_dir, *relative_parts))
+        return path if path.startswith("/Workspace") else "/Workspace" + path
+    else:
+        return os.path.join(*relative_parts)
+
+csv_file = get_absolute_path("..", "fixtures", "mycsv.csv")
+df = pd.read_csv(csv_file)
+display(df)
+```
diff --git a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/pyproject.toml.tmpl b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/pyproject.toml.tmpl
new file mode 100644
index 000000000..cee0d8946
--- /dev/null
+++ b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/pyproject.toml.tmpl
@@ -0,0 +1,57 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "{{.project_name}}"
+requires-python = ">=3.10"
+description = "wheel file based on {{.project_name}}"
+
+# Dependencies in case the output wheel file is used as a library dependency.
+# For defining dependencies, when this package is used in Databricks, see:
+# https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
+#
+# Example:
+# dependencies = [
+#     "requests==x.y.z",
+# ]
+dependencies = [
+]
+
+# see setup.py
+dynamic = ["version"]
+
+{{ if .include_python -}}
+[project.entry-points.packages]
+main = "{{.project_name}}.main:main"
+
+{{ end -}}
+
+[tool.setuptools]
+{{ if .include_python -}}
+py-modules = ["resources", "{{.project_name}}"]
+
+{{ else }}
+py-modules = ["resources"]
+
+{{ end -}}
+[tool.uv]
+## Dependencies for local development
+dev-dependencies = [
+    "databricks-bundles==0.7.0",
+
+    ## Add code completion support for DLT
+    # "databricks-dlt",
+
+    ## databricks-connect can be used to run parts of this project locally.
+    ## See https://docs.databricks.com/dev-tools/databricks-connect.html.
+    ##
+    ## Uncomment line below to install a version of db-connect that corresponds to
+    ## the Databricks Runtime version used for this project.
+    # "databricks-connect{{template "latest_lts_db_connect_version_spec"}}",
+]
+
+override-dependencies = [
+    # pyspark package conflicts with 'databricks-connect'
+    "pyspark; sys_platform == 'never'",
+]
diff --git a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/__init__.py b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/__init__.py
new file mode 100644
index 000000000..fbcb9dc5f
--- /dev/null
+++ b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/__init__.py
@@ -0,0 +1,16 @@
+from databricks.bundles.core import (
+    Bundle,
+    Resources,
+    load_resources_from_current_package_module,
+)
+
+
+def load_resources(bundle: Bundle) -> Resources:
+    """
+    'load_resources' function is referenced in databricks.yml and is responsible for loading
+    bundle resources defined in Python code. This function is called by Databricks CLI during
+    bundle deployment. After deployment, this function is not used.
+    """
+
+    # the default implementation loads all Python files in 'resources' directory
+    return load_resources_from_current_package_module()
diff --git a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/{{.project_name}}_job.py.tmpl b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/{{.project_name}}_job.py.tmpl
new file mode 100644
index 000000000..7c7a0d33f
--- /dev/null
+++ b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/{{.project_name}}_job.py.tmpl
@@ -0,0 +1,108 @@
+{{$include_dlt := "no" -}}
+from databricks.bundles.jobs import Job
+
+"""
+The main job for {{.project_name}}.
+
+{{- /* Clarify what this job is for for DLT-only users. */}}
+{{if and (eq $include_dlt "yes") (and (eq .include_notebook "no") (eq .include_python "no")) -}}
+This job runs {{.project_name}}_pipeline on a schedule.
+{{end -}}
+"""
+
+
+{{.project_name}}_job = Job.from_dict(
+    {
+        "name": "{{.project_name}}_job",
+        "trigger": {
+            # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger
+            "periodic": {
+                "interval": 1,
+                "unit": "DAYS",
+            },
+        },
+        {{- if not is_service_principal}}
+        "email_notifications": {
+            "on_failure": [
+                "{{user_name}}",
+            ],
+        },
+        {{else}}
+        {{- end -}}
+        "tasks": [
+            {{- if eq .include_notebook "yes" -}}
+            {{- "\n            " -}}
+            {
+                "task_key": "notebook_task",
+                "job_cluster_key": "job_cluster",
+                "notebook_task": {
+                    "notebook_path": "src/notebook.ipynb",
+                },
+            },
+            {{- end -}}
+            {{- if (eq $include_dlt "yes") -}}
+            {{- "\n            " -}}
+            {
+                "task_key": "refresh_pipeline",
+                {{- if (eq .include_notebook "yes" )}}
+                "depends_on": [
+                    {
+                        "task_key": "notebook_task",
+                    },
+                ],
+                {{- end}}
+                "pipeline_task": {
+                    {{- /* TODO: we should find a way that doesn't use magics for the below, like ./{{project_name}}.pipeline.yml */}}
+                    "pipeline_id": "${resources.pipelines.{{.project_name}}_pipeline.id}",
+                },
+            },
+            {{- end -}}
+            {{- if (eq .include_python "yes") -}}
+            {{- "\n            " -}}
+            {
+                "task_key": "main_task",
+                {{- if (eq $include_dlt "yes") }}
+                "depends_on": [
+                    {
+                        "task_key": "refresh_pipeline",
+                    },
+                ],
+                {{- else if (eq .include_notebook "yes" )}}
+                "depends_on": [
+                    {
+                        "task_key": "notebook_task",
+                    },
+                ],
+                {{- end}}
+                "job_cluster_key": "job_cluster",
+                "python_wheel_task": {
+                    "package_name": "{{.project_name}}",
+                    "entry_point": "main",
+                },
+                "libraries": [
+                    # By default we just include the .whl file generated for the {{.project_name}} package.
+                    # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
+                    # for more information on how to add other libraries.
+                    {
+                        "whl": "dist/*.whl",
+                    },
+                ],
+            },
+            {{- end -}}
+            {{""}}
+        ],
+        "job_clusters": [
+            {
+                "job_cluster_key": "job_cluster",
+                "new_cluster": {
+                    "spark_version": "{{template "latest_lts_dbr_version"}}",
+                    "node_type_id": "{{smallest_node_type}}",
+                    "autoscale": {
+                        "min_workers": 1,
+                        "max_workers": 4,
+                    },
+                },
+            },
+        ],
+    }
+)
diff --git a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/{{.project_name}}_pipeline.py.tmpl b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/{{.project_name}}_pipeline.py.tmpl
new file mode 100644
index 000000000..c8579ae65
--- /dev/null
+++ b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/{{.project_name}}_pipeline.py.tmpl
@@ -0,0 +1,24 @@
+from databricks.bundles.pipelines import Pipeline
+
+{{.project_name}}_pipeline = Pipeline.from_dict(
+    {
+        "name": "{{.project_name}}_pipeline",
+        "target": "{{.project_name}}_${bundle.target}",
+        {{- if or (eq default_catalog "") (eq default_catalog "hive_metastore")}}
+        ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog:
+        "catalog": "catalog_name",
+        {{- else}}
+        "catalog": "{{default_catalog}}",
+        {{- end}}
+        "libraries": [
+            {
+                "notebook": {
+                    "path": "src/dlt_pipeline.ipynb",
+                },
+            },
+        ],
+        "configuration": {
+            "bundle.sourcePath": "${workspace.file_path}/src",
+        },
+    }
+)
diff --git a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/scratch/README.md b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/scratch/README.md
new file mode 100644
index 000000000..e6cfb81b4
--- /dev/null
+++ b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/scratch/README.md
@@ -0,0 +1,4 @@
+# scratch
+
+This folder is reserved for personal, exploratory notebooks.
+By default these are not committed to Git, as 'scratch' is listed in .gitignore.
diff --git a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/setup.py.tmpl b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/setup.py.tmpl
new file mode 100644
index 000000000..19c9d0ebe
--- /dev/null
+++ b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/setup.py.tmpl
@@ -0,0 +1,18 @@
+"""
+setup.py configuration script describing how to build and package this project.
+
+This file is primarily used by the setuptools library and typically should not
+be executed directly. See README.md for how to deploy, test, and run
+the {{.project_name}} project.
+"""
+
+import os
+
+from setuptools import setup
+
+local_version = os.getenv("LOCAL_VERSION")
+version = "0.0.1"
+
+setup(
+    version=f"{version}+{local_version}" if local_version else version,
+)
diff --git a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl
new file mode 100644
index 000000000..629106dbf
--- /dev/null
+++ b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl
@@ -0,0 +1,104 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "9a626959-61c8-4bba-84d2-2a4ecab1f7ec",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "source": [
+    "# DLT pipeline\n",
+    "\n",
+    "This Delta Live Tables (DLT) definition is executed using a pipeline defined in resources/{{.project_name}}.pipeline.yml."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "9198e987-5606-403d-9f6d-8f14e6a4017f",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+   {{- if (eq .include_python "yes") }}
+    "# Import DLT and src/{{.project_name}}\n",
+    "import dlt\n",
+    "import sys\n",
+    "\n",
+    "sys.path.append(spark.conf.get(\"bundle.sourcePath\", \".\"))\n",
+    "from pyspark.sql.functions import expr\n",
+    "from {{.project_name}} import main"
+   {{else}}
+    "import dlt\n",
+    "from pyspark.sql.functions import expr\n",
+    "from pyspark.sql import SparkSession\n",
+    "\n",
+    "spark = SparkSession.builder.getOrCreate()"
+   {{end -}}
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "3fc19dba-61fd-4a89-8f8c-24fee63bfb14",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    {{- if (eq .include_python "yes") }}
+    "@dlt.view\n",
+    "def taxi_raw():\n",
+    "    return main.get_taxis(spark)\n",
+    {{else}}
+    "@dlt.view\n",
+    "def taxi_raw():\n",
+    "    return spark.read.format(\"json\").load(\"/databricks-datasets/nyctaxi/sample/json/\")\n",
+    {{end -}}
+    "\n",
+    "\n",
+    "@dlt.table\n",
+    "def filtered_taxis():\n",
+    "    return dlt.read(\"taxi_raw\").filter(expr(\"fare_amount < 30\"))"
+   ]
+  }
+ ],
+ "metadata": {
+  "application/vnd.databricks.v1+notebook": {
+   "dashboards": [],
+   "language": "python",
+   "notebookMetadata": {
+    "pythonIndentUnit": 2
+   },
+   "notebookName": "dlt_pipeline",
+   "widgets": {}
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/src/notebook.ipynb.tmpl b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/src/notebook.ipynb.tmpl
new file mode 100644
index 000000000..6782a053b
--- /dev/null
+++ b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/src/notebook.ipynb.tmpl
@@ -0,0 +1,79 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "ee353e42-ff58-4955-9608-12865bd0950e",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "source": [
+    "# Default notebook\n",
+    "\n",
+    "This default notebook is executed using Databricks Workflows as defined in resources/{{.project_name}}.job.yml."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "6bca260b-13d1-448f-8082-30b60a85c9ae",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+   {{- if (eq .include_python "yes") }}
+    "from {{.project_name}} import main\n",
+    "\n",
+    "main.get_taxis(spark).show(10)"
+   {{else}}
+    "spark.range(10)"
+   {{end -}}
+   ]
+  }
+ ],
+ "metadata": {
+  "application/vnd.databricks.v1+notebook": {
+   "dashboards": [],
+   "language": "python",
+   "notebookMetadata": {
+    "pythonIndentUnit": 2
+   },
+   "notebookName": "notebook",
+   "widgets": {}
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/src/{{.project_name}}/__init__.py.tmpl b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/src/{{.project_name}}/__init__.py.tmpl
new file mode 100644
index 000000000..e69de29bb
diff --git a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/src/{{.project_name}}/main.py.tmpl b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/src/{{.project_name}}/main.py.tmpl
new file mode 100644
index 000000000..5ae344c7e
--- /dev/null
+++ b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/src/{{.project_name}}/main.py.tmpl
@@ -0,0 +1,25 @@
+from pyspark.sql import SparkSession, DataFrame
+
+
+def get_taxis(spark: SparkSession) -> DataFrame:
+    return spark.read.table("samples.nyctaxi.trips")
+
+
+# Create a new Databricks Connect session. If this fails,
+# check that you have configured Databricks Connect correctly.
+# See https://docs.databricks.com/dev-tools/databricks-connect.html.
+def get_spark() -> SparkSession:
+    try:
+        from databricks.connect import DatabricksSession
+
+        return DatabricksSession.builder.getOrCreate()
+    except ImportError:
+        return SparkSession.builder.getOrCreate()
+
+
+def main():
+    get_taxis(get_spark()).show(5)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/tests/main_test.py.tmpl b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/tests/main_test.py.tmpl
new file mode 100644
index 000000000..6f89fca53
--- /dev/null
+++ b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/tests/main_test.py.tmpl
@@ -0,0 +1,8 @@
+from {{.project_name}}.main import get_taxis, get_spark
+
+# running tests requires installing databricks-connect, e.g. by uncommenting it in pyproject.toml
+
+
+def test_main():
+    taxis = get_taxis(get_spark())
+    assert taxis.count() > 5

From 41a21af556bc29edae8c3d2fc41584edef2c037a Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Mon, 20 Jan 2025 17:39:28 +0530
Subject: [PATCH 111/247] Refactor `bundle init` (#2074)

## Summary of changes
This PR introduces three new abstractions:
1. `Resolver`: Resolves which reader and writer to use for a template.
2. `Writer`: Writes a template project to disk. Prompts the user if
necessary.
3. `Reader`: Reads a template specification from disk, built into the
CLI or from GitHub.

Introducing these abstractions helps decouple reading a template from
writing it. When I tried adding telemetry for the `bundle init` command,
I noticed that the code in `cmd/init.go` was getting convoluted and hard
to test. A future change could have accidentally logged PII when a user
initialised a custom template.

Hedging against that risk is important here because we use a generic
untyped `map<string, string>` representation in the backend to log
telemetry for the `databricks bundle init`. Otherwise, we risk
accidentally breaking our compliance with our centralization
requirements.

### Details

After this PR there are two classes of templates that can be
initialized:
1. A `databricks` template: This could be a builtin template or a
template outside the CLI like mlops-stacks, which is still owned and
managed by Databricks. These templates log their telemetry arguments and
template name.
2. A `custom` template: These are templates created by and managed by
the end user. In these templates we do not log the template name and
args. Instead a generic placeholder string of "custom" is logged in our
telemetry system.

NOTE: The functionality of the `databricks bundle init` command remains
the same after this PR. Only the internal abstractions used are changed.

## Tests
New unit tests. Existing golden and unit tests. Also a fair bit of
manual testing.
---
 bundle/run/app_test.go                        |   6 +-
 bundle/run/job_test.go                        |   9 +-
 bundle/run/pipeline_test.go                   |   5 +-
 cmd/bundle/init.go                            | 253 ++----------------
 integration/bundle/helpers_test.go            |  14 +-
 libs/cmdio/io.go                              |  11 +
 .../databrickscfg/cfgpickers/clusters_test.go |   9 +-
 .../workspace_files_extensions_client.go      |  22 +-
 .../workspace_files_extensions_client_test.go |   2 +-
 libs/template/builtin.go                      |  12 +-
 libs/template/builtin_test.go                 |   4 +-
 libs/template/materialize.go                  |  94 -------
 libs/template/materialize_test.go             |  23 --
 libs/template/reader.go                       | 119 ++++++++
 libs/template/reader_test.go                  | 101 +++++++
 libs/template/resolver.go                     | 122 +++++++++
 libs/template/resolver_test.go                | 110 ++++++++
 libs/template/template.go                     | 140 ++++++++++
 .../template/template_test.go                 |  76 ++++--
 libs/template/writer.go                       | 171 ++++++++++++
 libs/template/writer_test.go                  |  58 ++++
 21 files changed, 939 insertions(+), 422 deletions(-)
 delete mode 100644 libs/template/materialize.go
 delete mode 100644 libs/template/materialize_test.go
 create mode 100644 libs/template/reader.go
 create mode 100644 libs/template/reader_test.go
 create mode 100644 libs/template/resolver.go
 create mode 100644 libs/template/resolver_test.go
 create mode 100644 libs/template/template.go
 rename cmd/bundle/init_test.go => libs/template/template_test.go (59%)
 create mode 100644 libs/template/writer.go
 create mode 100644 libs/template/writer_test.go

diff --git a/bundle/run/app_test.go b/bundle/run/app_test.go
index 8e82f45ae..77f197e8d 100644
--- a/bundle/run/app_test.go
+++ b/bundle/run/app_test.go
@@ -1,7 +1,6 @@
 package run
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"os"
@@ -16,7 +15,6 @@ import (
 	"github.com/databricks/cli/bundle/internal/bundletest"
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/cli/libs/dyn"
-	"github.com/databricks/cli/libs/flags"
 	"github.com/databricks/cli/libs/vfs"
 	"github.com/databricks/databricks-sdk-go/experimental/mocks"
 	"github.com/databricks/databricks-sdk-go/service/apps"
@@ -76,9 +74,7 @@ func setupBundle(t *testing.T) (context.Context, *bundle.Bundle, *mocks.MockWork
 	b.SetWorkpaceClient(mwc.WorkspaceClient)
 	bundletest.SetLocation(b, "resources.apps.my_app", []dyn.Location{{File: "./databricks.yml"}})
 
-	ctx := context.Background()
-	ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputText, &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}, "", "..."))
-	ctx = cmdio.NewContext(ctx, cmdio.NewLogger(flags.ModeAppend))
+	ctx := cmdio.MockDiscard(context.Background())
 
 	diags := bundle.Apply(ctx, b, bundle.Seq(
 		mutator.DefineDefaultWorkspacePaths(),
diff --git a/bundle/run/job_test.go b/bundle/run/job_test.go
index 72aecc887..daf6cf063 100644
--- a/bundle/run/job_test.go
+++ b/bundle/run/job_test.go
@@ -1,7 +1,6 @@
 package run
 
 import (
-	"bytes"
 	"context"
 	"testing"
 	"time"
@@ -159,8 +158,8 @@ func TestJobRunnerRestart(t *testing.T) {
 
 		m := mocks.NewMockWorkspaceClient(t)
 		b.SetWorkpaceClient(m.WorkspaceClient)
-		ctx := context.Background()
-		ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputText, &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}, "", ""))
+
+		ctx := cmdio.MockDiscard(context.Background())
 		ctx = cmdio.NewContext(ctx, cmdio.NewLogger(flags.ModeAppend))
 
 		jobApi := m.GetMockJobsAPI()
@@ -230,8 +229,8 @@ func TestJobRunnerRestartForContinuousUnpausedJobs(t *testing.T) {
 
 	m := mocks.NewMockWorkspaceClient(t)
 	b.SetWorkpaceClient(m.WorkspaceClient)
-	ctx := context.Background()
-	ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputText, &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}, "", "..."))
+
+	ctx := cmdio.MockDiscard(context.Background())
 	ctx = cmdio.NewContext(ctx, cmdio.NewLogger(flags.ModeAppend))
 
 	jobApi := m.GetMockJobsAPI()
diff --git a/bundle/run/pipeline_test.go b/bundle/run/pipeline_test.go
index bfa0c5846..56d800d35 100644
--- a/bundle/run/pipeline_test.go
+++ b/bundle/run/pipeline_test.go
@@ -1,7 +1,6 @@
 package run
 
 import (
-	"bytes"
 	"context"
 	"testing"
 	"time"
@@ -75,8 +74,8 @@ func TestPipelineRunnerRestart(t *testing.T) {
 		Host: "https://test.com",
 	}
 	b.SetWorkpaceClient(m.WorkspaceClient)
-	ctx := context.Background()
-	ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputText, &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}, "", "..."))
+
+	ctx := cmdio.MockDiscard(context.Background())
 	ctx = cmdio.NewContext(ctx, cmdio.NewLogger(flags.ModeAppend))
 
 	mockWait := &pipelines.WaitGetPipelineIdle[struct{}]{
diff --git a/cmd/bundle/init.go b/cmd/bundle/init.go
index 6b93fd1e5..1911abe19 100644
--- a/cmd/bundle/init.go
+++ b/cmd/bundle/init.go
@@ -1,180 +1,15 @@
 package bundle
 
 import (
-	"context"
 	"errors"
 	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-	"slices"
-	"strings"
 
 	"github.com/databricks/cli/cmd/root"
 	"github.com/databricks/cli/libs/cmdio"
-	"github.com/databricks/cli/libs/dbr"
-	"github.com/databricks/cli/libs/filer"
-	"github.com/databricks/cli/libs/git"
 	"github.com/databricks/cli/libs/template"
 	"github.com/spf13/cobra"
 )
 
-var gitUrlPrefixes = []string{
-	"https://",
-	"git@",
-}
-
-type nativeTemplate struct {
-	name        string
-	gitUrl      string
-	description string
-	aliases     []string
-	hidden      bool
-}
-
-const customTemplate = "custom..."
-
-var nativeTemplates = []nativeTemplate{
-	{
-		name:        "default-python",
-		description: "The default Python template for Notebooks / Delta Live Tables / Workflows",
-	},
-	{
-		name:        "default-sql",
-		description: "The default SQL template for .sql files that run with Databricks SQL",
-	},
-	{
-		name:        "dbt-sql",
-		description: "The dbt SQL template (databricks.com/blog/delivering-cost-effective-data-real-time-dbt-and-databricks)",
-	},
-	{
-		name:        "mlops-stacks",
-		gitUrl:      "https://github.com/databricks/mlops-stacks",
-		description: "The Databricks MLOps Stacks template (github.com/databricks/mlops-stacks)",
-		aliases:     []string{"mlops-stack"},
-	},
-	{
-		name:        "default-pydabs",
-		gitUrl:      "https://databricks.github.io/workflows-authoring-toolkit/pydabs-template.git",
-		hidden:      true,
-		description: "The default PyDABs template",
-	},
-	{
-		name:        "experimental-jobs-as-code",
-		hidden:      true,
-		description: "Jobs as code template (experimental)",
-	},
-	{
-		name:        customTemplate,
-		description: "Bring your own template",
-	},
-}
-
-// Return template descriptions for command-line help
-func nativeTemplateHelpDescriptions() string {
-	var lines []string
-	for _, template := range nativeTemplates {
-		if template.name != customTemplate && !template.hidden {
-			lines = append(lines, fmt.Sprintf("- %s: %s", template.name, template.description))
-		}
-	}
-	return strings.Join(lines, "\n")
-}
-
-// Return template options for an interactive prompt
-func nativeTemplateOptions() []cmdio.Tuple {
-	names := make([]cmdio.Tuple, 0, len(nativeTemplates))
-	for _, template := range nativeTemplates {
-		if template.hidden {
-			continue
-		}
-		tuple := cmdio.Tuple{
-			Name: template.name,
-			Id:   template.description,
-		}
-		names = append(names, tuple)
-	}
-	return names
-}
-
-func getNativeTemplateByDescription(description string) string {
-	for _, template := range nativeTemplates {
-		if template.description == description {
-			return template.name
-		}
-	}
-	return ""
-}
-
-func getUrlForNativeTemplate(name string) string {
-	for _, template := range nativeTemplates {
-		if template.name == name {
-			return template.gitUrl
-		}
-		if slices.Contains(template.aliases, name) {
-			return template.gitUrl
-		}
-	}
-	return ""
-}
-
-func getFsForNativeTemplate(name string) (fs.FS, error) {
-	builtin, err := template.Builtin()
-	if err != nil {
-		return nil, err
-	}
-
-	// If this is a built-in template, the return value will be non-nil.
-	var templateFS fs.FS
-	for _, entry := range builtin {
-		if entry.Name == name {
-			templateFS = entry.FS
-			break
-		}
-	}
-
-	return templateFS, nil
-}
-
-func isRepoUrl(url string) bool {
-	result := false
-	for _, prefix := range gitUrlPrefixes {
-		if strings.HasPrefix(url, prefix) {
-			result = true
-			break
-		}
-	}
-	return result
-}
-
-// Computes the repo name from the repo URL. Treats the last non empty word
-// when splitting at '/' as the repo name. For example: for url git@github.com:databricks/cli.git
-// the name would be "cli.git"
-func repoName(url string) string {
-	parts := strings.Split(strings.TrimRight(url, "/"), "/")
-	return parts[len(parts)-1]
-}
-
-func constructOutputFiler(ctx context.Context, outputDir string) (filer.Filer, error) {
-	outputDir, err := filepath.Abs(outputDir)
-	if err != nil {
-		return nil, err
-	}
-
-	// If the CLI is running on DBR and we're writing to the workspace file system,
-	// use the extension-aware workspace filesystem filer to instantiate the template.
-	//
-	// It is not possible to write notebooks through the workspace filesystem's FUSE mount.
-	// Therefore this is the only way we can initialize templates that contain notebooks
-	// when running the CLI on DBR and initializing a template to the workspace.
-	//
-	if strings.HasPrefix(outputDir, "/Workspace/") && dbr.RunsOnRuntime(ctx) {
-		return filer.NewWorkspaceFilesExtensionsClient(root.WorkspaceClient(ctx), outputDir)
-	}
-
-	return filer.NewLocalClient(outputDir)
-}
-
 func newInitCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "init [TEMPLATE_PATH]",
@@ -187,7 +22,7 @@ TEMPLATE_PATH optionally specifies which template to use. It can be one of the f
 - a local file system path with a template directory
 - a Git repository URL, e.g. https://github.com/my/repository
 
-See https://docs.databricks.com/en/dev-tools/bundles/templates.html for more information on templates.`, nativeTemplateHelpDescriptions()),
+See https://docs.databricks.com/en/dev-tools/bundles/templates.html for more information on templates.`, template.HelpDescriptions()),
 	}
 
 	var configFile string
@@ -207,88 +42,32 @@ See https://docs.databricks.com/en/dev-tools/bundles/templates.html for more inf
 			return errors.New("only one of --tag or --branch can be specified")
 		}
 
-		// Git ref to use for template initialization
-		ref := branch
-		if tag != "" {
-			ref = tag
+		var templatePathOrUrl string
+		if len(args) > 0 {
+			templatePathOrUrl = args[0]
+		}
+		r := template.Resolver{
+			TemplatePathOrUrl: templatePathOrUrl,
+			ConfigFile:        configFile,
+			OutputDir:         outputDir,
+			TemplateDir:       templateDir,
+			Tag:               tag,
+			Branch:            branch,
 		}
 
 		ctx := cmd.Context()
-		var templatePath string
-		if len(args) > 0 {
-			templatePath = args[0]
-		} else {
-			var err error
-			if !cmdio.IsPromptSupported(ctx) {
-				return errors.New("please specify a template")
-			}
-			description, err := cmdio.SelectOrdered(ctx, nativeTemplateOptions(), "Template to use")
-			if err != nil {
-				return err
-			}
-			templatePath = getNativeTemplateByDescription(description)
-		}
-
-		outputFiler, err := constructOutputFiler(ctx, outputDir)
-		if err != nil {
-			return err
-		}
-
-		if templatePath == customTemplate {
+		tmpl, err := r.Resolve(ctx)
+		if errors.Is(err, template.ErrCustomSelected) {
 			cmdio.LogString(ctx, "Please specify a path or Git repository to use a custom template.")
 			cmdio.LogString(ctx, "See https://docs.databricks.com/en/dev-tools/bundles/templates.html to learn more about custom templates.")
 			return nil
 		}
-
-		// Expand templatePath to a git URL if it's an alias for a known native template
-		// and we know it's git URL.
-		if gitUrl := getUrlForNativeTemplate(templatePath); gitUrl != "" {
-			templatePath = gitUrl
-		}
-
-		if !isRepoUrl(templatePath) {
-			if templateDir != "" {
-				return errors.New("--template-dir can only be used with a Git repository URL")
-			}
-
-			templateFS, err := getFsForNativeTemplate(templatePath)
-			if err != nil {
-				return err
-			}
-
-			// If this is not a built-in template, then it must be a local file system path.
-			if templateFS == nil {
-				templateFS = os.DirFS(templatePath)
-			}
-
-			// skip downloading the repo because input arg is not a URL. We assume
-			// it's a path on the local file system in that case
-			return template.Materialize(ctx, configFile, templateFS, outputFiler)
-		}
-
-		// Create a temporary directory with the name of the repository.  The '*'
-		// character is replaced by a random string in the generated temporary directory.
-		repoDir, err := os.MkdirTemp("", repoName(templatePath)+"-*")
 		if err != nil {
 			return err
 		}
+		defer tmpl.Reader.Cleanup(ctx)
 
-		// start the spinner
-		promptSpinner := cmdio.Spinner(ctx)
-		promptSpinner <- "Downloading the template\n"
-
-		// TODO: Add automated test that the downloaded git repo is cleaned up.
-		// Clone the repository in the temporary directory
-		err = git.Clone(ctx, templatePath, ref, repoDir)
-		close(promptSpinner)
-		if err != nil {
-			return err
-		}
-
-		// Clean up downloaded repository once the template is materialized.
-		defer os.RemoveAll(repoDir)
-		templateFS := os.DirFS(filepath.Join(repoDir, templateDir))
-		return template.Materialize(ctx, configFile, templateFS, outputFiler)
+		return tmpl.Writer.Materialize(ctx, tmpl.Reader)
 	}
 	return cmd
 }
diff --git a/integration/bundle/helpers_test.go b/integration/bundle/helpers_test.go
index a537ca351..b4f9c9086 100644
--- a/integration/bundle/helpers_test.go
+++ b/integration/bundle/helpers_test.go
@@ -16,7 +16,6 @@ import (
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/cli/libs/env"
-	"github.com/databricks/cli/libs/filer"
 	"github.com/databricks/cli/libs/flags"
 	"github.com/databricks/cli/libs/folders"
 	"github.com/databricks/cli/libs/template"
@@ -40,10 +39,19 @@ func initTestTemplateWithBundleRoot(t testutil.TestingT, ctx context.Context, te
 	cmd := cmdio.NewIO(ctx, flags.OutputJSON, strings.NewReader(""), os.Stdout, os.Stderr, "", "bundles")
 	ctx = cmdio.InContext(ctx, cmd)
 
-	out, err := filer.NewLocalClient(bundleRoot)
+	r := template.Resolver{
+		TemplatePathOrUrl: templateRoot,
+		ConfigFile:        configFilePath,
+		OutputDir:         bundleRoot,
+	}
+
+	tmpl, err := r.Resolve(ctx)
 	require.NoError(t, err)
-	err = template.Materialize(ctx, configFilePath, os.DirFS(templateRoot), out)
+	defer tmpl.Reader.Cleanup(ctx)
+
+	err = tmpl.Writer.Materialize(ctx, tmpl.Reader)
 	require.NoError(t, err)
+
 	return bundleRoot
 }
 
diff --git a/libs/cmdio/io.go b/libs/cmdio/io.go
index c0e9e868a..11b75157d 100644
--- a/libs/cmdio/io.go
+++ b/libs/cmdio/io.go
@@ -285,3 +285,14 @@ func fromContext(ctx context.Context) *cmdIO {
 	}
 	return io
 }
+
+// Mocks the context with a cmdio object that discards all output.
+func MockDiscard(ctx context.Context) context.Context {
+	return InContext(ctx, &cmdIO{
+		interactive:  false,
+		outputFormat: flags.OutputText,
+		in:           io.NopCloser(strings.NewReader("")),
+		out:          io.Discard,
+		err:          io.Discard,
+	})
+}
diff --git a/libs/databrickscfg/cfgpickers/clusters_test.go b/libs/databrickscfg/cfgpickers/clusters_test.go
index 29e190a93..840916e91 100644
--- a/libs/databrickscfg/cfgpickers/clusters_test.go
+++ b/libs/databrickscfg/cfgpickers/clusters_test.go
@@ -1,12 +1,10 @@
 package cfgpickers
 
 import (
-	"bytes"
 	"context"
 	"testing"
 
 	"github.com/databricks/cli/libs/cmdio"
-	"github.com/databricks/cli/libs/flags"
 	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/qa"
 	"github.com/databricks/databricks-sdk-go/service/compute"
@@ -114,8 +112,8 @@ func TestFirstCompatibleCluster(t *testing.T) {
 	defer server.Close()
 	w := databricks.Must(databricks.NewWorkspaceClient((*databricks.Config)(cfg)))
 
-	ctx := context.Background()
-	ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputText, &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}, "", "..."))
+	ctx := cmdio.MockDiscard(context.Background())
+
 	clusterID, err := AskForCluster(ctx, w, WithDatabricksConnect("13.1"))
 	require.NoError(t, err)
 	require.Equal(t, "bcd-id", clusterID)
@@ -161,8 +159,7 @@ func TestNoCompatibleClusters(t *testing.T) {
 	defer server.Close()
 	w := databricks.Must(databricks.NewWorkspaceClient((*databricks.Config)(cfg)))
 
-	ctx := context.Background()
-	ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputText, &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}, "", "..."))
+	ctx := cmdio.MockDiscard(context.Background())
 	_, err := AskForCluster(ctx, w, WithDatabricksConnect("13.1"))
 	require.Equal(t, ErrNoCompatibleClusters, err)
 }
diff --git a/libs/filer/workspace_files_extensions_client.go b/libs/filer/workspace_files_extensions_client.go
index 9ee2722e1..0127d180c 100644
--- a/libs/filer/workspace_files_extensions_client.go
+++ b/libs/filer/workspace_files_extensions_client.go
@@ -16,7 +16,7 @@ import (
 	"github.com/databricks/databricks-sdk-go/service/workspace"
 )
 
-type workspaceFilesExtensionsClient struct {
+type WorkspaceFilesExtensionsClient struct {
 	workspaceClient *databricks.WorkspaceClient
 
 	wsfs     Filer
@@ -32,7 +32,7 @@ type workspaceFileStatus struct {
 	nameForWorkspaceAPI string
 }
 
-func (w *workspaceFilesExtensionsClient) stat(ctx context.Context, name string) (wsfsFileInfo, error) {
+func (w *WorkspaceFilesExtensionsClient) stat(ctx context.Context, name string) (wsfsFileInfo, error) {
 	info, err := w.wsfs.Stat(ctx, name)
 	if err != nil {
 		return wsfsFileInfo{}, err
@@ -42,7 +42,7 @@ func (w *workspaceFilesExtensionsClient) stat(ctx context.Context, name string)
 
 // This function returns the stat for the provided notebook. The stat object itself contains the path
 // with the extension since it is meant to be used in the context of a fs.FileInfo.
-func (w *workspaceFilesExtensionsClient) getNotebookStatByNameWithExt(ctx context.Context, name string) (*workspaceFileStatus, error) {
+func (w *WorkspaceFilesExtensionsClient) getNotebookStatByNameWithExt(ctx context.Context, name string) (*workspaceFileStatus, error) {
 	ext := path.Ext(name)
 	nameWithoutExt := strings.TrimSuffix(name, ext)
 
@@ -104,7 +104,7 @@ func (w *workspaceFilesExtensionsClient) getNotebookStatByNameWithExt(ctx contex
 	}, nil
 }
 
-func (w *workspaceFilesExtensionsClient) getNotebookStatByNameWithoutExt(ctx context.Context, name string) (*workspaceFileStatus, error) {
+func (w *WorkspaceFilesExtensionsClient) getNotebookStatByNameWithoutExt(ctx context.Context, name string) (*workspaceFileStatus, error) {
 	stat, err := w.stat(ctx, name)
 	if err != nil {
 		return nil, err
@@ -184,7 +184,7 @@ func newWorkspaceFilesExtensionsClient(w *databricks.WorkspaceClient, root strin
 		filer = newWorkspaceFilesReadaheadCache(filer)
 	}
 
-	return &workspaceFilesExtensionsClient{
+	return &WorkspaceFilesExtensionsClient{
 		workspaceClient: w,
 
 		wsfs:     filer,
@@ -193,7 +193,7 @@ func newWorkspaceFilesExtensionsClient(w *databricks.WorkspaceClient, root strin
 	}, nil
 }
 
-func (w *workspaceFilesExtensionsClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) {
+func (w *WorkspaceFilesExtensionsClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) {
 	entries, err := w.wsfs.ReadDir(ctx, name)
 	if err != nil {
 		return nil, err
@@ -235,7 +235,7 @@ func (w *workspaceFilesExtensionsClient) ReadDir(ctx context.Context, name strin
 // Note: The import API returns opaque internal errors for namespace clashes
 // (e.g. a file and a notebook or a directory and a notebook). Thus users of this
 // method should be careful to avoid such clashes.
-func (w *workspaceFilesExtensionsClient) Write(ctx context.Context, name string, reader io.Reader, mode ...WriteMode) error {
+func (w *WorkspaceFilesExtensionsClient) Write(ctx context.Context, name string, reader io.Reader, mode ...WriteMode) error {
 	if w.readonly {
 		return ReadOnlyError{"write"}
 	}
@@ -244,7 +244,7 @@ func (w *workspaceFilesExtensionsClient) Write(ctx context.Context, name string,
 }
 
 // Try to read the file as a regular file. If the file is not found, try to read it as a notebook.
-func (w *workspaceFilesExtensionsClient) Read(ctx context.Context, name string) (io.ReadCloser, error) {
+func (w *WorkspaceFilesExtensionsClient) Read(ctx context.Context, name string) (io.ReadCloser, error) {
 	// Ensure that the file / notebook exists. We do this check here to avoid reading
 	// the content of a notebook called `foo` when the user actually wanted
 	// to read the content of a file called `foo`.
@@ -283,7 +283,7 @@ func (w *workspaceFilesExtensionsClient) Read(ctx context.Context, name string)
 }
 
 // Try to delete the file as a regular file. If the file is not found, try to delete it as a notebook.
-func (w *workspaceFilesExtensionsClient) Delete(ctx context.Context, name string, mode ...DeleteMode) error {
+func (w *WorkspaceFilesExtensionsClient) Delete(ctx context.Context, name string, mode ...DeleteMode) error {
 	if w.readonly {
 		return ReadOnlyError{"delete"}
 	}
@@ -320,7 +320,7 @@ func (w *workspaceFilesExtensionsClient) Delete(ctx context.Context, name string
 }
 
 // Try to stat the file as a regular file. If the file is not found, try to stat it as a notebook.
-func (w *workspaceFilesExtensionsClient) Stat(ctx context.Context, name string) (fs.FileInfo, error) {
+func (w *WorkspaceFilesExtensionsClient) Stat(ctx context.Context, name string) (fs.FileInfo, error) {
 	info, err := w.wsfs.Stat(ctx, name)
 
 	// If the file is not found, it might be a notebook.
@@ -361,7 +361,7 @@ func (w *workspaceFilesExtensionsClient) Stat(ctx context.Context, name string)
 // Note: The import API returns opaque internal errors for namespace clashes
 // (e.g. a file and a notebook or a directory and a notebook). Thus users of this
 // method should be careful to avoid such clashes.
-func (w *workspaceFilesExtensionsClient) Mkdir(ctx context.Context, name string) error {
+func (w *WorkspaceFilesExtensionsClient) Mkdir(ctx context.Context, name string) error {
 	if w.readonly {
 		return ReadOnlyError{"mkdir"}
 	}
diff --git a/libs/filer/workspace_files_extensions_client_test.go b/libs/filer/workspace_files_extensions_client_test.go
index 10a2bebf0..9ea837fa9 100644
--- a/libs/filer/workspace_files_extensions_client_test.go
+++ b/libs/filer/workspace_files_extensions_client_test.go
@@ -181,7 +181,7 @@ func TestFilerWorkspaceFilesExtensionsErrorsOnDupName(t *testing.T) {
 				root:            NewWorkspaceRootPath("/dir"),
 			}
 
-			workspaceFilesExtensionsClient := workspaceFilesExtensionsClient{
+			workspaceFilesExtensionsClient := WorkspaceFilesExtensionsClient{
 				workspaceClient: mockedWorkspaceClient.WorkspaceClient,
 				wsfs:            &workspaceFilesClient,
 			}
diff --git a/libs/template/builtin.go b/libs/template/builtin.go
index dcb3a8858..5b10534ef 100644
--- a/libs/template/builtin.go
+++ b/libs/template/builtin.go
@@ -8,14 +8,14 @@ import (
 //go:embed all:templates
 var builtinTemplates embed.FS
 
-// BuiltinTemplate represents a template that is built into the CLI.
-type BuiltinTemplate struct {
+// builtinTemplate represents a template that is built into the CLI.
+type builtinTemplate struct {
 	Name string
 	FS   fs.FS
 }
 
-// Builtin returns the list of all built-in templates.
-func Builtin() ([]BuiltinTemplate, error) {
+// builtin returns the list of all built-in templates.
+func builtin() ([]builtinTemplate, error) {
 	templates, err := fs.Sub(builtinTemplates, "templates")
 	if err != nil {
 		return nil, err
@@ -26,7 +26,7 @@ func Builtin() ([]BuiltinTemplate, error) {
 		return nil, err
 	}
 
-	var out []BuiltinTemplate
+	var out []builtinTemplate
 	for _, entry := range entries {
 		if !entry.IsDir() {
 			continue
@@ -37,7 +37,7 @@ func Builtin() ([]BuiltinTemplate, error) {
 			return nil, err
 		}
 
-		out = append(out, BuiltinTemplate{
+		out = append(out, builtinTemplate{
 			Name: entry.Name(),
 			FS:   templateFS,
 		})
diff --git a/libs/template/builtin_test.go b/libs/template/builtin_test.go
index 79e04cb84..162a227ea 100644
--- a/libs/template/builtin_test.go
+++ b/libs/template/builtin_test.go
@@ -9,12 +9,12 @@ import (
 )
 
 func TestBuiltin(t *testing.T) {
-	out, err := Builtin()
+	out, err := builtin()
 	require.NoError(t, err)
 	assert.GreaterOrEqual(t, len(out), 3)
 
 	// Create a map of templates by name for easier lookup
-	templates := make(map[string]*BuiltinTemplate)
+	templates := make(map[string]*builtinTemplate)
 	for _, tmpl := range out {
 		templates[tmpl.Name] = &tmpl
 	}
diff --git a/libs/template/materialize.go b/libs/template/materialize.go
deleted file mode 100644
index 86a6a8c37..000000000
--- a/libs/template/materialize.go
+++ /dev/null
@@ -1,94 +0,0 @@
-package template
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io/fs"
-
-	"github.com/databricks/cli/libs/cmdio"
-	"github.com/databricks/cli/libs/filer"
-)
-
-const (
-	libraryDirName  = "library"
-	templateDirName = "template"
-	schemaFileName  = "databricks_template_schema.json"
-)
-
-// This function materializes the input templates as a project, using user defined
-// configurations.
-// Parameters:
-//
-//	ctx: 			context containing a cmdio object. This is used to prompt the user
-//	configFilePath: file path containing user defined config values
-//	templateFS: 	root of the template definition
-//	outputFiler: 	filer to use for writing the initialized template
-func Materialize(ctx context.Context, configFilePath string, templateFS fs.FS, outputFiler filer.Filer) error {
-	if _, err := fs.Stat(templateFS, schemaFileName); errors.Is(err, fs.ErrNotExist) {
-		return fmt.Errorf("not a bundle template: expected to find a template schema file at %s", schemaFileName)
-	}
-
-	config, err := newConfig(ctx, templateFS, schemaFileName)
-	if err != nil {
-		return err
-	}
-
-	// Read and assign config values from file
-	if configFilePath != "" {
-		err = config.assignValuesFromFile(configFilePath)
-		if err != nil {
-			return err
-		}
-	}
-
-	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, config.values, helpers, templateFS, templateDirName, libraryDirName)
-	if err != nil {
-		return err
-	}
-
-	// Print welcome message
-	welcome := config.schema.WelcomeMessage
-	if welcome != "" {
-		welcome, err = r.executeTemplate(welcome)
-		if err != nil {
-			return err
-		}
-		cmdio.LogString(ctx, welcome)
-	}
-
-	// Prompt user for any missing config values. Assign default values if
-	// terminal is not TTY
-	err = config.promptOrAssignDefaultValues(r)
-	if err != nil {
-		return err
-	}
-	err = config.validate()
-	if err != nil {
-		return err
-	}
-
-	// Walk and render the template, since input configuration is complete
-	err = r.walk()
-	if err != nil {
-		return err
-	}
-
-	err = r.persistToDisk(ctx, outputFiler)
-	if err != nil {
-		return err
-	}
-
-	success := config.schema.SuccessMessage
-	if success == "" {
-		cmdio.LogString(ctx, "✨ Successfully initialized template")
-	} else {
-		success, err = r.executeTemplate(success)
-		if err != nil {
-			return err
-		}
-		cmdio.LogString(ctx, success)
-	}
-	return nil
-}
diff --git a/libs/template/materialize_test.go b/libs/template/materialize_test.go
deleted file mode 100644
index c9331b43f..000000000
--- a/libs/template/materialize_test.go
+++ /dev/null
@@ -1,23 +0,0 @@
-package template
-
-import (
-	"context"
-	"os"
-	"testing"
-
-	"github.com/databricks/cli/cmd/root"
-	"github.com/databricks/databricks-sdk-go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestMaterializeForNonTemplateDirectory(t *testing.T) {
-	tmpDir := t.TempDir()
-	w, err := databricks.NewWorkspaceClient(&databricks.Config{})
-	require.NoError(t, err)
-	ctx := root.SetWorkspaceClient(context.Background(), w)
-
-	// Try to materialize a non-template directory.
-	err = Materialize(ctx, "", os.DirFS(tmpDir), nil)
-	assert.EqualError(t, err, "not a bundle template: expected to find a template schema file at "+schemaFileName)
-}
diff --git a/libs/template/reader.go b/libs/template/reader.go
new file mode 100644
index 000000000..8e32a75cf
--- /dev/null
+++ b/libs/template/reader.go
@@ -0,0 +1,119 @@
+package template
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/log"
+)
+
+type Reader interface {
+	// FS returns a file system that contains the template
+	// definition files.
+	FS(ctx context.Context) (fs.FS, error)
+
+	// Cleanup releases any resources associated with the reader
+	// like cleaning up temporary directories.
+	Cleanup(ctx context.Context)
+}
+
+type builtinReader struct {
+	name string
+}
+
+func (r *builtinReader) FS(ctx context.Context) (fs.FS, error) {
+	builtin, err := builtin()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, entry := range builtin {
+		if entry.Name == r.name {
+			return entry.FS, nil
+		}
+	}
+
+	return nil, fmt.Errorf("builtin template %s not found", r.name)
+}
+
+func (r *builtinReader) Cleanup(ctx context.Context) {}
+
+type gitReader struct {
+	gitUrl string
+	// tag or branch to checkout
+	ref string
+	// subdirectory within the repository that contains the template
+	templateDir string
+	// temporary directory where the repository is cloned
+	tmpRepoDir string
+
+	// Function to clone the repository. This is a function pointer to allow
+	// mocking in tests.
+	cloneFunc func(ctx context.Context, url, reference, targetPath string) error
+}
+
+// Computes the repo name from the repo URL. Treats the last non empty word
+// when splitting at '/' as the repo name. For example: for url git@github.com:databricks/cli.git
+// the name would be "cli.git"
+func repoName(url string) string {
+	parts := strings.Split(strings.TrimRight(url, "/"), "/")
+	return parts[len(parts)-1]
+}
+
+func (r *gitReader) FS(ctx context.Context) (fs.FS, error) {
+	// Calling FS twice will lead to two downloaded copies of the git repo.
+	// In the future if you need to call FS twice, consider adding some caching
+	// logic here to avoid multiple downloads.
+	if r.tmpRepoDir != "" {
+		return nil, errors.New("FS called twice on git reader")
+	}
+
+	// Create a temporary directory with the name of the repository.  The '*'
+	// character is replaced by a random string in the generated temporary directory.
+	repoDir, err := os.MkdirTemp("", repoName(r.gitUrl)+"-*")
+	if err != nil {
+		return nil, err
+	}
+	r.tmpRepoDir = repoDir
+
+	// start the spinner
+	promptSpinner := cmdio.Spinner(ctx)
+	promptSpinner <- "Downloading the template\n"
+
+	err = r.cloneFunc(ctx, r.gitUrl, r.ref, repoDir)
+	close(promptSpinner)
+	if err != nil {
+		return nil, err
+	}
+
+	return os.DirFS(filepath.Join(repoDir, r.templateDir)), nil
+}
+
+func (r *gitReader) Cleanup(ctx context.Context) {
+	if r.tmpRepoDir == "" {
+		return
+	}
+
+	// Cleanup is best effort. Only log errors.
+	err := os.RemoveAll(r.tmpRepoDir)
+	if err != nil {
+		log.Debugf(ctx, "Error cleaning up tmp directory %s for git template reader for URL %s: %s", r.tmpRepoDir, r.gitUrl, err)
+	}
+}
+
+type localReader struct {
+	// Path on the local filesystem that contains the template
+	path string
+}
+
+func (r *localReader) FS(ctx context.Context) (fs.FS, error) {
+	return os.DirFS(r.path), nil
+}
+
+func (r *localReader) Cleanup(ctx context.Context) {}
diff --git a/libs/template/reader_test.go b/libs/template/reader_test.go
new file mode 100644
index 000000000..89c8d9fdf
--- /dev/null
+++ b/libs/template/reader_test.go
@@ -0,0 +1,101 @@
+package template
+
+import (
+	"context"
+	"io/fs"
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/cli/internal/testutil"
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBuiltInReader(t *testing.T) {
+	exists := []string{
+		"default-python",
+		"default-sql",
+		"dbt-sql",
+		"experimental-jobs-as-code",
+	}
+
+	for _, name := range exists {
+		t.Run(name, func(t *testing.T) {
+			r := &builtinReader{name: name}
+			fsys, err := r.FS(context.Background())
+			assert.NoError(t, err)
+			assert.NotNil(t, fsys)
+
+			// Assert file content returned is accurate and every template has a welcome
+			// message defined.
+			b, err := fs.ReadFile(fsys, "databricks_template_schema.json")
+			require.NoError(t, err)
+			assert.Contains(t, string(b), "welcome_message")
+		})
+	}
+
+	t.Run("doesnotexist", func(t *testing.T) {
+		r := &builtinReader{name: "doesnotexist"}
+		_, err := r.FS(context.Background())
+		assert.EqualError(t, err, "builtin template doesnotexist not found")
+	})
+}
+
+func TestGitUrlReader(t *testing.T) {
+	ctx := cmdio.MockDiscard(context.Background())
+
+	var args []string
+	numCalls := 0
+	cloneFunc := func(ctx context.Context, url, reference, targetPath string) error {
+		numCalls++
+		args = []string{url, reference, targetPath}
+		testutil.WriteFile(t, filepath.Join(targetPath, "a", "b", "c", "somefile"), "somecontent")
+		return nil
+	}
+	r := &gitReader{
+		gitUrl:      "someurl",
+		cloneFunc:   cloneFunc,
+		ref:         "sometag",
+		templateDir: "a/b/c",
+	}
+
+	// Assert cloneFunc is called with the correct args.
+	fsys, err := r.FS(ctx)
+	require.NoError(t, err)
+	require.NotEmpty(t, r.tmpRepoDir)
+	assert.Equal(t, 1, numCalls)
+	assert.DirExists(t, r.tmpRepoDir)
+	assert.Equal(t, []string{"someurl", "sometag", r.tmpRepoDir}, args)
+
+	// Assert the fs returned is rooted at the templateDir.
+	b, err := fs.ReadFile(fsys, "somefile")
+	require.NoError(t, err)
+	assert.Equal(t, "somecontent", string(b))
+
+	// Assert second call to FS returns an error.
+	_, err = r.FS(ctx)
+	assert.ErrorContains(t, err, "FS called twice on git reader")
+
+	// Assert the downloaded repository is cleaned up.
+	_, err = fs.Stat(fsys, ".")
+	require.NoError(t, err)
+	r.Cleanup(ctx)
+	_, err = fs.Stat(fsys, ".")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+}
+
+func TestLocalReader(t *testing.T) {
+	tmpDir := t.TempDir()
+	testutil.WriteFile(t, filepath.Join(tmpDir, "somefile"), "somecontent")
+	ctx := context.Background()
+
+	r := &localReader{path: tmpDir}
+	fsys, err := r.FS(ctx)
+	require.NoError(t, err)
+
+	// Assert the fs returned is rooted at correct location.
+	b, err := fs.ReadFile(fsys, "somefile")
+	require.NoError(t, err)
+	assert.Equal(t, "somecontent", string(b))
+}
diff --git a/libs/template/resolver.go b/libs/template/resolver.go
new file mode 100644
index 000000000..2cc8bf1c7
--- /dev/null
+++ b/libs/template/resolver.go
@@ -0,0 +1,122 @@
+package template
+
+import (
+	"context"
+	"errors"
+	"strings"
+
+	"github.com/databricks/cli/libs/git"
+)
+
+var gitUrlPrefixes = []string{
+	"https://",
+	"git@",
+}
+
+func isRepoUrl(url string) bool {
+	result := false
+	for _, prefix := range gitUrlPrefixes {
+		if strings.HasPrefix(url, prefix) {
+			result = true
+			break
+		}
+	}
+	return result
+}
+
+type Resolver struct {
+	// One of the following three:
+	// 1. Path to a local template directory.
+	// 2. URL to a Git repository containing a template.
+	// 3. Name of a built-in template.
+	TemplatePathOrUrl string
+
+	// Path to a JSON file containing the configuration values to be used for
+	// template initialization.
+	ConfigFile string
+
+	// Directory to write the initialized template to.
+	OutputDir string
+
+	// Directory path within a Git repository containing the template.
+	TemplateDir string
+
+	// Git tag or branch to download the template from. Only one of these can be
+	// specified.
+	Tag    string
+	Branch string
+}
+
+// ErrCustomSelected is returned when the user selects the "custom..." option
+// in the prompt UI when they run `databricks bundle init`. This error signals
+// the upstream callsite to show documentation to the user on how to use a custom
+// template.
+var ErrCustomSelected = errors.New("custom template selected")
+
+// Configures the reader and the writer for template and returns
+// a handle to the template.
+// Prompts the user if needed.
+func (r Resolver) Resolve(ctx context.Context) (*Template, error) {
+	if r.Tag != "" && r.Branch != "" {
+		return nil, errors.New("only one of tag or branch can be specified")
+	}
+
+	// Git ref to use for template initialization
+	ref := r.Branch
+	if r.Tag != "" {
+		ref = r.Tag
+	}
+
+	var err error
+	var templateName TemplateName
+
+	if r.TemplatePathOrUrl == "" {
+		// Prompt the user to select a template
+		// if a template path or URL is not provided.
+		templateName, err = SelectTemplate(ctx)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		templateName = TemplateName(r.TemplatePathOrUrl)
+	}
+
+	tmpl := GetDatabricksTemplate(templateName)
+
+	// If we could not find a databricks template with the name provided by the user,
+	// then we assume that the user provided us with a reference to a custom template.
+	//
+	// This reference could be one of:
+	// 1. Path to a local template directory.
+	// 2. URL to a Git repository containing a template.
+	//
+	// We resolve the appropriate reader according to the reference provided by the user.
+	if tmpl == nil {
+		tmpl = &Template{
+			name: Custom,
+			// We use a writer that does not log verbose telemetry for custom templates.
+			// This is important because template definitions can contain PII that we
+			// do not want to centralize.
+			Writer: &defaultWriter{},
+		}
+
+		if isRepoUrl(r.TemplatePathOrUrl) {
+			tmpl.Reader = &gitReader{
+				gitUrl:      r.TemplatePathOrUrl,
+				ref:         ref,
+				templateDir: r.TemplateDir,
+				cloneFunc:   git.Clone,
+			}
+		} else {
+			tmpl.Reader = &localReader{
+				path: r.TemplatePathOrUrl,
+			}
+		}
+	}
+	err = tmpl.Writer.Configure(ctx, r.ConfigFile, r.OutputDir)
+	if err != nil {
+		return nil, err
+	}
+
+	return tmpl, nil
+}
diff --git a/libs/template/resolver_test.go b/libs/template/resolver_test.go
new file mode 100644
index 000000000..1dee1c45f
--- /dev/null
+++ b/libs/template/resolver_test.go
@@ -0,0 +1,110 @@
+package template
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTemplateResolverBothTagAndBranch(t *testing.T) {
+	r := Resolver{
+		Tag:    "tag",
+		Branch: "branch",
+	}
+
+	_, err := r.Resolve(context.Background())
+	assert.EqualError(t, err, "only one of tag or branch can be specified")
+}
+
+func TestTemplateResolverErrorsWhenPromptingIsNotSupported(t *testing.T) {
+	r := Resolver{}
+	ctx := cmdio.MockDiscard(context.Background())
+
+	_, err := r.Resolve(ctx)
+	assert.EqualError(t, err, "prompting is not supported. Please specify the path, name or URL of the template to use")
+}
+
+func TestTemplateResolverForDefaultTemplates(t *testing.T) {
+	for _, name := range []string{
+		"default-python",
+		"default-sql",
+		"dbt-sql",
+	} {
+		t.Run(name, func(t *testing.T) {
+			r := Resolver{
+				TemplatePathOrUrl: name,
+			}
+
+			tmpl, err := r.Resolve(context.Background())
+			require.NoError(t, err)
+
+			assert.Equal(t, &builtinReader{name: name}, tmpl.Reader)
+			assert.IsType(t, &writerWithFullTelemetry{}, tmpl.Writer)
+		})
+	}
+
+	t.Run("mlops-stacks", func(t *testing.T) {
+		r := Resolver{
+			TemplatePathOrUrl: "mlops-stacks",
+			ConfigFile:        "/config/file",
+		}
+
+		tmpl, err := r.Resolve(context.Background())
+		require.NoError(t, err)
+
+		// Assert reader and writer configuration
+		assert.Equal(t, "https://github.com/databricks/mlops-stacks", tmpl.Reader.(*gitReader).gitUrl)
+		assert.Equal(t, "/config/file", tmpl.Writer.(*writerWithFullTelemetry).configPath)
+	})
+}
+
+func TestTemplateResolverForCustomUrl(t *testing.T) {
+	r := Resolver{
+		TemplatePathOrUrl: "https://www.example.com/abc",
+		Tag:               "tag",
+		TemplateDir:       "/template/dir",
+		ConfigFile:        "/config/file",
+	}
+
+	tmpl, err := r.Resolve(context.Background())
+	require.NoError(t, err)
+
+	assert.Equal(t, Custom, tmpl.name)
+
+	// Assert reader configuration
+	assert.Equal(t, "https://www.example.com/abc", tmpl.Reader.(*gitReader).gitUrl)
+	assert.Equal(t, "tag", tmpl.Reader.(*gitReader).ref)
+	assert.Equal(t, "/template/dir", tmpl.Reader.(*gitReader).templateDir)
+
+	// Assert writer configuration
+	assert.Equal(t, "/config/file", tmpl.Writer.(*defaultWriter).configPath)
+}
+
+func TestTemplateResolverForCustomPath(t *testing.T) {
+	r := Resolver{
+		TemplatePathOrUrl: "/custom/path",
+		ConfigFile:        "/config/file",
+	}
+
+	tmpl, err := r.Resolve(context.Background())
+	require.NoError(t, err)
+
+	assert.Equal(t, Custom, tmpl.name)
+
+	// Assert reader configuration
+	assert.Equal(t, "/custom/path", tmpl.Reader.(*localReader).path)
+
+	// Assert writer configuration
+	assert.Equal(t, "/config/file", tmpl.Writer.(*defaultWriter).configPath)
+}
+
+func TestBundleInitIsRepoUrl(t *testing.T) {
+	assert.True(t, isRepoUrl("git@github.com:databricks/cli.git"))
+	assert.True(t, isRepoUrl("https://github.com/databricks/cli.git"))
+
+	assert.False(t, isRepoUrl("./local"))
+	assert.False(t, isRepoUrl("foo"))
+}
diff --git a/libs/template/template.go b/libs/template/template.go
new file mode 100644
index 000000000..44834436b
--- /dev/null
+++ b/libs/template/template.go
@@ -0,0 +1,140 @@
+package template
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/git"
+)
+
+type Template struct {
+	Reader Reader
+	Writer Writer
+
+	name        TemplateName
+	description string
+	aliases     []string
+	hidden      bool
+}
+
+type TemplateName string
+
+const (
+	DefaultPython          TemplateName = "default-python"
+	DefaultSql             TemplateName = "default-sql"
+	DbtSql                 TemplateName = "dbt-sql"
+	MlopsStacks            TemplateName = "mlops-stacks"
+	DefaultPydabs          TemplateName = "default-pydabs"
+	Custom                 TemplateName = "custom"
+	ExperimentalJobsAsCode TemplateName = "experimental-jobs-as-code"
+)
+
+var databricksTemplates = []Template{
+	{
+		name:        DefaultPython,
+		description: "The default Python template for Notebooks / Delta Live Tables / Workflows",
+		Reader:      &builtinReader{name: string(DefaultPython)},
+		Writer:      &writerWithFullTelemetry{},
+	},
+	{
+		name:        DefaultSql,
+		description: "The default SQL template for .sql files that run with Databricks SQL",
+		Reader:      &builtinReader{name: string(DefaultSql)},
+		Writer:      &writerWithFullTelemetry{},
+	},
+	{
+		name:        DbtSql,
+		description: "The dbt SQL template (databricks.com/blog/delivering-cost-effective-data-real-time-dbt-and-databricks)",
+		Reader:      &builtinReader{name: string(DbtSql)},
+		Writer:      &writerWithFullTelemetry{},
+	},
+	{
+		name:        MlopsStacks,
+		description: "The Databricks MLOps Stacks template (github.com/databricks/mlops-stacks)",
+		aliases:     []string{"mlops-stack"},
+		Reader:      &gitReader{gitUrl: "https://github.com/databricks/mlops-stacks", cloneFunc: git.Clone},
+		Writer:      &writerWithFullTelemetry{},
+	},
+	{
+		name:        DefaultPydabs,
+		hidden:      true,
+		description: "The default PyDABs template",
+		Reader:      &gitReader{gitUrl: "https://databricks.github.io/workflows-authoring-toolkit/pydabs-template.git", cloneFunc: git.Clone},
+		Writer:      &writerWithFullTelemetry{},
+	},
+	{
+		name:        ExperimentalJobsAsCode,
+		hidden:      true,
+		description: "Jobs as code template (experimental)",
+		Reader:      &builtinReader{name: string(ExperimentalJobsAsCode)},
+		Writer:      &writerWithFullTelemetry{},
+	},
+}
+
+func HelpDescriptions() string {
+	var lines []string
+	for _, template := range databricksTemplates {
+		if template.name != Custom && !template.hidden {
+			lines = append(lines, fmt.Sprintf("- %s: %s", template.name, template.description))
+		}
+	}
+	return strings.Join(lines, "\n")
+}
+
+var customTemplateDescription = "Bring your own template"
+
+func options() []cmdio.Tuple {
+	names := make([]cmdio.Tuple, 0, len(databricksTemplates))
+	for _, template := range databricksTemplates {
+		if template.hidden {
+			continue
+		}
+		tuple := cmdio.Tuple{
+			Name: string(template.name),
+			Id:   template.description,
+		}
+		names = append(names, tuple)
+	}
+
+	names = append(names, cmdio.Tuple{
+		Name: "custom...",
+		Id:   customTemplateDescription,
+	})
+	return names
+}
+
+func SelectTemplate(ctx context.Context) (TemplateName, error) {
+	if !cmdio.IsPromptSupported(ctx) {
+		return "", errors.New("prompting is not supported. Please specify the path, name or URL of the template to use")
+	}
+	description, err := cmdio.SelectOrdered(ctx, options(), "Template to use")
+	if err != nil {
+		return "", err
+	}
+
+	if description == customTemplateDescription {
+		return TemplateName(""), ErrCustomSelected
+	}
+
+	for _, template := range databricksTemplates {
+		if template.description == description {
+			return template.name, nil
+		}
+	}
+
+	return "", fmt.Errorf("template with description %s not found", description)
+}
+
+func GetDatabricksTemplate(name TemplateName) *Template {
+	for _, template := range databricksTemplates {
+		if template.name == name || slices.Contains(template.aliases, string(name)) {
+			return &template
+		}
+	}
+
+	return nil
+}
diff --git a/cmd/bundle/init_test.go b/libs/template/template_test.go
similarity index 59%
rename from cmd/bundle/init_test.go
rename to libs/template/template_test.go
index 475b2e149..80391e58b 100644
--- a/cmd/bundle/init_test.go
+++ b/libs/template/template_test.go
@@ -1,4 +1,4 @@
-package bundle
+package template
 
 import (
 	"testing"
@@ -7,12 +7,23 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestBundleInitIsRepoUrl(t *testing.T) {
-	assert.True(t, isRepoUrl("git@github.com:databricks/cli.git"))
-	assert.True(t, isRepoUrl("https://github.com/databricks/cli.git"))
+func TestTemplateHelpDescriptions(t *testing.T) {
+	expected := `- default-python: The default Python template for Notebooks / Delta Live Tables / Workflows
+- default-sql: The default SQL template for .sql files that run with Databricks SQL
+- dbt-sql: The dbt SQL template (databricks.com/blog/delivering-cost-effective-data-real-time-dbt-and-databricks)
+- mlops-stacks: The Databricks MLOps Stacks template (github.com/databricks/mlops-stacks)`
+	assert.Equal(t, expected, HelpDescriptions())
+}
 
-	assert.False(t, isRepoUrl("./local"))
-	assert.False(t, isRepoUrl("foo"))
+func TestTemplateOptions(t *testing.T) {
+	expected := []cmdio.Tuple{
+		{Name: "default-python", Id: "The default Python template for Notebooks / Delta Live Tables / Workflows"},
+		{Name: "default-sql", Id: "The default SQL template for .sql files that run with Databricks SQL"},
+		{Name: "dbt-sql", Id: "The dbt SQL template (databricks.com/blog/delivering-cost-effective-data-real-time-dbt-and-databricks)"},
+		{Name: "mlops-stacks", Id: "The Databricks MLOps Stacks template (github.com/databricks/mlops-stacks)"},
+		{Name: "custom...", Id: "Bring your own template"},
+	}
+	assert.Equal(t, expected, options())
 }
 
 func TestBundleInitRepoName(t *testing.T) {
@@ -27,28 +38,41 @@ func TestBundleInitRepoName(t *testing.T) {
 	assert.Equal(t, "www.github.com", repoName("https://www.github.com"))
 }
 
-func TestNativeTemplateOptions(t *testing.T) {
-	expected := []cmdio.Tuple{
-		{Name: "default-python", Id: "The default Python template for Notebooks / Delta Live Tables / Workflows"},
-		{Name: "default-sql", Id: "The default SQL template for .sql files that run with Databricks SQL"},
-		{Name: "dbt-sql", Id: "The dbt SQL template (databricks.com/blog/delivering-cost-effective-data-real-time-dbt-and-databricks)"},
-		{Name: "mlops-stacks", Id: "The Databricks MLOps Stacks template (github.com/databricks/mlops-stacks)"},
-		{Name: "custom...", Id: "Bring your own template"},
+func TestTemplateTelemetryIsCapturedForAllDefaultTemplates(t *testing.T) {
+	for _, tmpl := range databricksTemplates {
+		w := tmpl.Writer
+
+		// Assert telemetry is captured for all databricks templates, i.e. templates
+		// owned by databricks.
+		assert.IsType(t, &writerWithFullTelemetry{}, w)
 	}
-	assert.Equal(t, expected, nativeTemplateOptions())
 }
 
-func TestNativeTemplateHelpDescriptions(t *testing.T) {
-	expected := `- default-python: The default Python template for Notebooks / Delta Live Tables / Workflows
-- default-sql: The default SQL template for .sql files that run with Databricks SQL
-- dbt-sql: The dbt SQL template (databricks.com/blog/delivering-cost-effective-data-real-time-dbt-and-databricks)
-- mlops-stacks: The Databricks MLOps Stacks template (github.com/databricks/mlops-stacks)`
-	assert.Equal(t, expected, nativeTemplateHelpDescriptions())
-}
+func TestTemplateGetDatabricksTemplate(t *testing.T) {
+	names := []TemplateName{
+		DefaultPython,
+		DefaultSql,
+		DbtSql,
+		MlopsStacks,
+		DefaultPydabs,
+	}
 
-func TestGetUrlForNativeTemplate(t *testing.T) {
-	assert.Equal(t, "https://github.com/databricks/mlops-stacks", getUrlForNativeTemplate("mlops-stacks"))
-	assert.Equal(t, "https://github.com/databricks/mlops-stacks", getUrlForNativeTemplate("mlops-stack"))
-	assert.Equal(t, "", getUrlForNativeTemplate("default-python"))
-	assert.Equal(t, "", getUrlForNativeTemplate("invalid"))
+	for _, name := range names {
+		tmpl := GetDatabricksTemplate(name)
+		assert.Equal(t, tmpl.name, name)
+	}
+
+	notExist := []string{
+		"/some/path",
+		"doesnotexist",
+		"https://www.someurl.com",
+	}
+
+	for _, name := range notExist {
+		tmpl := GetDatabricksTemplate(TemplateName(name))
+		assert.Nil(t, tmpl)
+	}
+
+	// Assert the alias works.
+	assert.Equal(t, MlopsStacks, GetDatabricksTemplate(TemplateName("mlops-stack")).name)
 }
diff --git a/libs/template/writer.go b/libs/template/writer.go
new file mode 100644
index 000000000..e3d5af583
--- /dev/null
+++ b/libs/template/writer.go
@@ -0,0 +1,171 @@
+package template
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"path/filepath"
+	"strings"
+
+	"github.com/databricks/cli/cmd/root"
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/dbr"
+	"github.com/databricks/cli/libs/filer"
+)
+
+const (
+	libraryDirName  = "library"
+	templateDirName = "template"
+	schemaFileName  = "databricks_template_schema.json"
+)
+
+type Writer interface {
+	// Configure the writer with:
+	// 1. The path to the config file (if any) that contains input values for the
+	// template.
+	// 2. The output directory where the template will be materialized.
+	Configure(ctx context.Context, configPath, outputDir string) error
+
+	// Materialize the template to the local file system.
+	Materialize(ctx context.Context, r Reader) error
+}
+
+type defaultWriter struct {
+	configPath  string
+	outputFiler filer.Filer
+
+	// Internal state
+	config   *config
+	renderer *renderer
+}
+
+func constructOutputFiler(ctx context.Context, outputDir string) (filer.Filer, error) {
+	outputDir, err := filepath.Abs(outputDir)
+	if err != nil {
+		return nil, err
+	}
+
+	// If the CLI is running on DBR and we're writing to the workspace file system,
+	// use the extension-aware workspace filesystem filer to instantiate the template.
+	//
+	// It is not possible to write notebooks through the workspace filesystem's FUSE mount.
+	// Therefore this is the only way we can initialize templates that contain notebooks
+	// when running the CLI on DBR and initializing a template to the workspace.
+	//
+	if strings.HasPrefix(outputDir, "/Workspace/") && dbr.RunsOnRuntime(ctx) {
+		return filer.NewWorkspaceFilesExtensionsClient(root.WorkspaceClient(ctx), outputDir)
+	}
+
+	return filer.NewLocalClient(outputDir)
+}
+
+func (tmpl *defaultWriter) Configure(ctx context.Context, configPath, outputDir string) error {
+	tmpl.configPath = configPath
+
+	outputFiler, err := constructOutputFiler(ctx, outputDir)
+	if err != nil {
+		return err
+	}
+
+	tmpl.outputFiler = outputFiler
+	return nil
+}
+
+func (tmpl *defaultWriter) promptForInput(ctx context.Context, reader Reader) error {
+	readerFs, err := reader.FS(ctx)
+	if err != nil {
+		return err
+	}
+	if _, err := fs.Stat(readerFs, schemaFileName); errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("not a bundle template: expected to find a template schema file at %s", schemaFileName)
+	}
+
+	tmpl.config, err = newConfig(ctx, readerFs, schemaFileName)
+	if err != nil {
+		return err
+	}
+
+	// Read and assign config values from file
+	if tmpl.configPath != "" {
+		err = tmpl.config.assignValuesFromFile(tmpl.configPath)
+		if err != nil {
+			return err
+		}
+	}
+
+	helpers := loadHelpers(ctx)
+	tmpl.renderer, err = newRenderer(ctx, tmpl.config.values, helpers, readerFs, templateDirName, libraryDirName)
+	if err != nil {
+		return err
+	}
+
+	// Print welcome message
+	welcome := tmpl.config.schema.WelcomeMessage
+	if welcome != "" {
+		welcome, err = tmpl.renderer.executeTemplate(welcome)
+		if err != nil {
+			return err
+		}
+		cmdio.LogString(ctx, welcome)
+	}
+
+	// Prompt user for any missing config values. Assign default values if
+	// terminal is not TTY
+	err = tmpl.config.promptOrAssignDefaultValues(tmpl.renderer)
+	if err != nil {
+		return err
+	}
+	return tmpl.config.validate()
+}
+
+func (tmpl *defaultWriter) printSuccessMessage(ctx context.Context) error {
+	success := tmpl.config.schema.SuccessMessage
+	if success == "" {
+		cmdio.LogString(ctx, "✨ Successfully initialized template")
+		return nil
+	}
+
+	success, err := tmpl.renderer.executeTemplate(success)
+	if err != nil {
+		return err
+	}
+	cmdio.LogString(ctx, success)
+	return nil
+}
+
+func (tmpl *defaultWriter) Materialize(ctx context.Context, reader Reader) error {
+	err := tmpl.promptForInput(ctx, reader)
+	if err != nil {
+		return err
+	}
+
+	// Walk the template file tree and compute in-memory representations of the
+	// output files.
+	err = tmpl.renderer.walk()
+	if err != nil {
+		return err
+	}
+
+	// Flush the output files to disk.
+	err = tmpl.renderer.persistToDisk(ctx, tmpl.outputFiler)
+	if err != nil {
+		return err
+	}
+
+	return tmpl.printSuccessMessage(ctx)
+}
+
+func (tmpl *defaultWriter) LogTelemetry(ctx context.Context) error {
+	// TODO, only log the template name and uuid.
+	return nil
+}
+
+type writerWithFullTelemetry struct {
+	defaultWriter
+}
+
+func (tmpl *writerWithFullTelemetry) LogTelemetry(ctx context.Context) error {
+	// TODO, log template name, uuid and enum args as well.
+	return nil
+}
diff --git a/libs/template/writer_test.go b/libs/template/writer_test.go
new file mode 100644
index 000000000..9d57966ee
--- /dev/null
+++ b/libs/template/writer_test.go
@@ -0,0 +1,58 @@
+package template
+
+import (
+	"context"
+	"runtime"
+	"testing"
+
+	"github.com/databricks/cli/cmd/root"
+	"github.com/databricks/cli/libs/dbr"
+	"github.com/databricks/cli/libs/filer"
+	"github.com/databricks/databricks-sdk-go"
+	workspaceConfig "github.com/databricks/databricks-sdk-go/config"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDefaultWriterConfigure(t *testing.T) {
+	// Test on local file system.
+	w := &defaultWriter{}
+	err := w.Configure(context.Background(), "/foo/bar", "/out/abc")
+	assert.NoError(t, err)
+
+	assert.Equal(t, "/foo/bar", w.configPath)
+	assert.IsType(t, &filer.LocalClient{}, w.outputFiler)
+}
+
+func TestDefaultWriterConfigureOnDBR(t *testing.T) {
+	// This test is not valid on windows because a DBR image is always based on
+	// Linux.
+	if runtime.GOOS == "windows" {
+		t.Skip("Skipping test on Windows")
+	}
+
+	ctx := dbr.MockRuntime(context.Background(), true)
+	ctx = root.SetWorkspaceClient(ctx, &databricks.WorkspaceClient{
+		Config: &workspaceConfig.Config{Host: "https://myhost.com"},
+	})
+	w := &defaultWriter{}
+	err := w.Configure(ctx, "/foo/bar", "/Workspace/out/abc")
+	assert.NoError(t, err)
+
+	assert.Equal(t, "/foo/bar", w.configPath)
+	assert.IsType(t, &filer.WorkspaceFilesExtensionsClient{}, w.outputFiler)
+}
+
+func TestMaterializeForNonTemplateDirectory(t *testing.T) {
+	tmpDir1 := t.TempDir()
+	tmpDir2 := t.TempDir()
+	ctx := context.Background()
+
+	w := &defaultWriter{}
+	err := w.Configure(ctx, "/foo/bar", tmpDir1)
+	require.NoError(t, err)
+
+	// Try to materialize a non-template directory.
+	err = w.Materialize(ctx, &localReader{path: tmpDir2})
+	assert.EqualError(t, err, "not a bundle template: expected to find a template schema file at databricks_template_schema.json")
+}

From e6982d09ac73e33aca9d127457faa44df3d9a0ac Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Mon, 20 Jan 2025 20:22:22 +0530
Subject: [PATCH 112/247] Add doc string for `bundle.uuid` (#2170)

Co-authored-by: Julia Crawford (Databricks) <julia.crawford@databricks.com>
---
 bundle/internal/schema/annotations.yml | 2 +-
 bundle/schema/jsonschema.json          | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml
index 28d29798a..e18de7896 100644
--- a/bundle/internal/schema/annotations.yml
+++ b/bundle/internal/schema/annotations.yml
@@ -55,7 +55,7 @@ github.com/databricks/cli/bundle/config.Bundle:
       The name of the bundle.
   "uuid":
     "description": |-
-      PLACEHOLDER
+      Reserved. A Universally Unique Identifier (UUID) for the bundle that uniquely identifies the bundle in internal Databricks systems. This is generated when a bundle project is initialized using a Databricks template (using the `databricks bundle init` command).
 github.com/databricks/cli/bundle/config.Deployment:
   "fail_on_active_runs":
     "description": |-
diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json
index f55813b06..57b0cd231 100644
--- a/bundle/schema/jsonschema.json
+++ b/bundle/schema/jsonschema.json
@@ -1126,6 +1126,7 @@
                       "$ref": "#/$defs/string"
                     },
                     "uuid": {
+                      "description": "Reserved. A Universally Unique Identifier (UUID) for the bundle that uniquely identifies the bundle in internal Databricks systems. This is generated when a bundle project is initialized using a Databricks template (using the `databricks bundle init` command).",
                       "$ref": "#/$defs/string"
                     }
                   },

From 395a04a8d1124617419f183a67d9be2c5bb02dcd Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 20 Jan 2025 16:41:24 +0100
Subject: [PATCH 113/247] Run tests with coverage on CI (#2141)

Combine 'make cover' and 'make acc-cover' into single command. They
still write coverage into different files -- it would be useful to see
separate coverage numbers.

Note, we're not making use of coverage information yet. However, running
tests in CI with coverage will
- let us catch issues that only manifest when coverage is enabled, like
https://github.com/databricks/cli/pull/2150
- will let us know if there are any issues with running coverage on CI
before investing in additional coverage support
---
 .github/workflows/push.yml |  4 ++--
 Makefile                   | 15 +++++++--------
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
index d3363b7e3..c3a314d69 100644
--- a/.github/workflows/push.yml
+++ b/.github/workflows/push.yml
@@ -77,8 +77,8 @@ jobs:
           make vendor
           pip3 install wheel
 
-      - name: Run tests
-        run: make test
+      - name: Run tests with coverage
+        run: make cover
 
   golangci:
     needs: cleanups
diff --git a/Makefile b/Makefile
index 4b66c9254..00dadcb0c 100644
--- a/Makefile
+++ b/Makefile
@@ -3,6 +3,8 @@ default: vendor fmt lint
 PACKAGES=./acceptance/... ./libs/... ./internal/... ./cmd/... ./bundle/... .
 
 GOTESTSUM_FORMAT ?= pkgname-and-test-fails
+GOTESTSUM_CMD ?= gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped
+
 
 lint:
 	golangci-lint run --fix
@@ -18,22 +20,19 @@ fmt:
 	golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix ./...
 
 test:
-	gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped -- ${PACKAGES}
+	${GOTESTSUM_CMD} -- ${PACKAGES}
 
 cover:
-	gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped -- -coverprofile=coverage.txt ${PACKAGES}
-
-showcover:
-	go tool cover -html=coverage.txt
-
-acc-cover:
 	rm -fr ./acceptance/build/cover/
-	CLI_GOCOVERDIR=build/cover go test ./acceptance
+	CLI_GOCOVERDIR=build/cover ${GOTESTSUM_CMD} -- -coverprofile=coverage.txt ${PACKAGES}
 	rm -fr ./acceptance/build/cover-merged/
 	mkdir -p acceptance/build/cover-merged/
 	go tool covdata merge -i $$(printf '%s,' acceptance/build/cover/* | sed 's/,$$//') -o acceptance/build/cover-merged/
 	go tool covdata textfmt -i acceptance/build/cover-merged -o coverage-acceptance.txt
 
+showcover:
+	go tool cover -html=coverage.txt
+
 acc-showcover:
 	go tool cover -html=coverage-acceptance.txt
 

From 69f3c0a86959063af2edd6c1505c68c879106c3d Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Mon, 20 Jan 2025 17:02:29 +0100
Subject: [PATCH 114/247] Fix duplicate "apps" entry in help output (#2191)

## Changes

This is not needed because the command group is already returned by
`workspace.All()`.

The additional command registration was added in #1679.

## Tests

Acceptance test.
---
 acceptance/help/output.txt | 1 -
 cmd/cmd.go                 | 2 --
 2 files changed, 3 deletions(-)

diff --git a/acceptance/help/output.txt b/acceptance/help/output.txt
index ed4a88ce6..18434251d 100644
--- a/acceptance/help/output.txt
+++ b/acceptance/help/output.txt
@@ -115,7 +115,6 @@ Marketplace
 
 Apps
   apps                                   Apps run directly on a customer’s Databricks instance, integrate with their data, use and extend Databricks services, and enable users to interact through single sign-on.
-  apps                                   Apps run directly on a customer’s Databricks instance, integrate with their data, use and extend Databricks services, and enable users to interact through single sign-on.
 
 Clean Rooms
   clean-room-assets                      Clean room assets are data and code objects — Tables, volumes, and notebooks that are shared with the clean room.
diff --git a/cmd/cmd.go b/cmd/cmd.go
index 5b53a4ae5..5d835409f 100644
--- a/cmd/cmd.go
+++ b/cmd/cmd.go
@@ -15,7 +15,6 @@ import (
 	"github.com/databricks/cli/cmd/sync"
 	"github.com/databricks/cli/cmd/version"
 	"github.com/databricks/cli/cmd/workspace"
-	"github.com/databricks/cli/cmd/workspace/apps"
 	"github.com/spf13/cobra"
 )
 
@@ -68,7 +67,6 @@ func New(ctx context.Context) *cobra.Command {
 
 	// Add other subcommands.
 	cli.AddCommand(api.New())
-	cli.AddCommand(apps.New())
 	cli.AddCommand(auth.New())
 	cli.AddCommand(bundle.New())
 	cli.AddCommand(configure.New())

From 84a73052d2f1d0b4d89df2d9287089996f7b9d9b Mon Sep 17 00:00:00 2001
From: Ilya Kuznetsov <ilya.kuznetsov@databricks.com>
Date: Mon, 20 Jan 2025 17:16:51 +0100
Subject: [PATCH 115/247] fix: Detailed message for using source-linked
 deployment with file_path specified (#2119)

## Changes

Resolves remaining comments from here
https://github.com/databricks/cli/pull/2046


[This](https://github.com/databricks/cli/pull/2046#discussion_r1907121844)
and
[this](https://github.com/databricks/cli/pull/2046#discussion_r1908928239)
are on hold until Pieter's response

## Tests
<!-- How is this tested? -->
---
 .../mutator/apply_source_linked_deployment_preset.go     | 9 +++++----
 bundle/deploy/metadata/compute.go                        | 1 +
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/bundle/config/mutator/apply_source_linked_deployment_preset.go b/bundle/config/mutator/apply_source_linked_deployment_preset.go
index 839648301..570ca72cf 100644
--- a/bundle/config/mutator/apply_source_linked_deployment_preset.go
+++ b/bundle/config/mutator/apply_source_linked_deployment_preset.go
@@ -72,17 +72,18 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle
 		return diags
 	}
 
+	// This mutator runs before workspace paths are defaulted so it's safe to check for the user-defined value
 	if b.Config.Workspace.FilePath != "" && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
-		path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("workspace"), dyn.Key("file_path"))
-
+		path := dyn.NewPath(dyn.Key("workspace"), dyn.Key("file_path"))
 		diags = diags.Append(
 			diag.Diagnostic{
 				Severity: diag.Warning,
 				Summary:  "workspace.file_path setting will be ignored in source-linked deployment mode",
+				Detail:   "In source-linked deployment files are not copied to the destination and resources use source files instead",
 				Paths: []dyn.Path{
-					path[2:],
+					path,
 				},
-				Locations: b.Config.GetLocations(path[2:].String()),
+				Locations: b.Config.GetLocations(path.String()),
 			},
 		)
 	}
diff --git a/bundle/deploy/metadata/compute.go b/bundle/deploy/metadata/compute.go
index b47baa6b2..633d97081 100644
--- a/bundle/deploy/metadata/compute.go
+++ b/bundle/deploy/metadata/compute.go
@@ -54,6 +54,7 @@ func (m *compute) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
 
 	// Set file upload destination of the bundle in metadata
 	b.Metadata.Config.Workspace.FilePath = b.Config.Workspace.FilePath
+	// In source-linked deployment files are not copied and resources use source files, therefore we use sync path as file path in metadata
 	if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
 		b.Metadata.Config.Workspace.FilePath = b.SyncRootPath
 	}

From ee4a4b4c248ebdc1db93fe854adc5b26d12c8ddf Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 20 Jan 2025 17:33:03 +0100
Subject: [PATCH 116/247] Migrate quality_monitor_test.go to acceptance test
 (#2192)

---
 .../bundle}/quality_monitor/databricks.yml    |  0
 acceptance/bundle/quality_monitor/output.txt  | 73 +++++++++++++++++++
 acceptance/bundle/quality_monitor/script      |  3 +
 bundle/tests/quality_monitor_test.go          | 59 ---------------
 4 files changed, 76 insertions(+), 59 deletions(-)
 rename {bundle/tests => acceptance/bundle}/quality_monitor/databricks.yml (100%)
 create mode 100644 acceptance/bundle/quality_monitor/output.txt
 create mode 100644 acceptance/bundle/quality_monitor/script
 delete mode 100644 bundle/tests/quality_monitor_test.go

diff --git a/bundle/tests/quality_monitor/databricks.yml b/acceptance/bundle/quality_monitor/databricks.yml
similarity index 100%
rename from bundle/tests/quality_monitor/databricks.yml
rename to acceptance/bundle/quality_monitor/databricks.yml
diff --git a/acceptance/bundle/quality_monitor/output.txt b/acceptance/bundle/quality_monitor/output.txt
new file mode 100644
index 000000000..b3718c802
--- /dev/null
+++ b/acceptance/bundle/quality_monitor/output.txt
@@ -0,0 +1,73 @@
+
+>>> $CLI bundle validate -o json -t development
+{
+  "mode": "development",
+  "quality_monitors": {
+    "my_monitor": {
+      "assets_dir": "/Shared/provider-test/databricks_monitoring/main.test.thing1",
+      "inference_log": {
+        "granularities": [
+          "1 day"
+        ],
+        "model_id_col": "model_id",
+        "prediction_col": "prediction",
+        "problem_type": "PROBLEM_TYPE_REGRESSION",
+        "timestamp_col": "timestamp"
+      },
+      "output_schema_name": "main.dev",
+      "schedule": null,
+      "table_name": "main.test.dev"
+    }
+  }
+}
+
+>>> $CLI bundle validate -o json -t staging
+{
+  "mode": null,
+  "quality_monitors": {
+    "my_monitor": {
+      "assets_dir": "/Shared/provider-test/databricks_monitoring/main.test.thing1",
+      "inference_log": {
+        "granularities": [
+          "1 day"
+        ],
+        "model_id_col": "model_id",
+        "prediction_col": "prediction",
+        "problem_type": "PROBLEM_TYPE_REGRESSION",
+        "timestamp_col": "timestamp"
+      },
+      "output_schema_name": "main.staging",
+      "schedule": {
+        "quartz_cron_expression": "0 0 12 * * ?",
+        "timezone_id": "UTC"
+      },
+      "table_name": "main.test.staging"
+    }
+  }
+}
+
+>>> $CLI bundle validate -o json -t production
+{
+  "mode": null,
+  "quality_monitors": {
+    "my_monitor": {
+      "assets_dir": "/Shared/provider-test/databricks_monitoring/main.test.thing1",
+      "inference_log": {
+        "granularities": [
+          "1 day",
+          "1 hour"
+        ],
+        "model_id_col": "model_id_prod",
+        "prediction_col": "prediction_prod",
+        "problem_type": "PROBLEM_TYPE_REGRESSION",
+        "timestamp_col": "timestamp_prod"
+      },
+      "output_schema_name": "main.prod",
+      "schedule": {
+        "quartz_cron_expression": "0 0 12 * * ?",
+        "timezone_id": "UTC"
+      },
+      "table_name": "main.test.prod"
+    }
+  }
+}
diff --git a/acceptance/bundle/quality_monitor/script b/acceptance/bundle/quality_monitor/script
new file mode 100644
index 000000000..85a69d5e7
--- /dev/null
+++ b/acceptance/bundle/quality_monitor/script
@@ -0,0 +1,3 @@
+trace $CLI bundle validate -o json -t development | jq '{ mode: .bundle.mode, quality_monitors: .resources.quality_monitors }'
+trace $CLI bundle validate -o json -t staging | jq '{ mode: .bundle.mode, quality_monitors: .resources.quality_monitors }'
+trace $CLI bundle validate -o json -t production | jq '{ mode: .bundle.mode, quality_monitors: .resources.quality_monitors }'
diff --git a/bundle/tests/quality_monitor_test.go b/bundle/tests/quality_monitor_test.go
deleted file mode 100644
index e95c7b7c1..000000000
--- a/bundle/tests/quality_monitor_test.go
+++ /dev/null
@@ -1,59 +0,0 @@
-package config_tests
-
-import (
-	"testing"
-
-	"github.com/databricks/cli/bundle/config"
-	"github.com/databricks/cli/bundle/config/resources"
-	"github.com/databricks/databricks-sdk-go/service/catalog"
-	"github.com/stretchr/testify/assert"
-)
-
-func assertExpectedMonitor(t *testing.T, p *resources.QualityMonitor) {
-	assert.Equal(t, "timestamp", p.InferenceLog.TimestampCol)
-	assert.Equal(t, "prediction", p.InferenceLog.PredictionCol)
-	assert.Equal(t, "model_id", p.InferenceLog.ModelIdCol)
-	assert.Equal(t, catalog.MonitorInferenceLogProblemType("PROBLEM_TYPE_REGRESSION"), p.InferenceLog.ProblemType)
-}
-
-func TestMonitorTableNames(t *testing.T) {
-	b := loadTarget(t, "./quality_monitor", "development")
-	assert.Len(t, b.Config.Resources.QualityMonitors, 1)
-	assert.Equal(t, config.Development, b.Config.Bundle.Mode)
-
-	p := b.Config.Resources.QualityMonitors["my_monitor"]
-	assert.Equal(t, "main.test.dev", p.TableName)
-	assert.Equal(t, "/Shared/provider-test/databricks_monitoring/main.test.thing1", p.AssetsDir)
-	assert.Equal(t, "main.dev", p.OutputSchemaName)
-
-	assertExpectedMonitor(t, p)
-}
-
-func TestMonitorStaging(t *testing.T) {
-	b := loadTarget(t, "./quality_monitor", "staging")
-	assert.Len(t, b.Config.Resources.QualityMonitors, 1)
-
-	p := b.Config.Resources.QualityMonitors["my_monitor"]
-	assert.Equal(t, "main.test.staging", p.TableName)
-	assert.Equal(t, "/Shared/provider-test/databricks_monitoring/main.test.thing1", p.AssetsDir)
-	assert.Equal(t, "main.staging", p.OutputSchemaName)
-
-	assertExpectedMonitor(t, p)
-}
-
-func TestMonitorProduction(t *testing.T) {
-	b := loadTarget(t, "./quality_monitor", "production")
-	assert.Len(t, b.Config.Resources.QualityMonitors, 1)
-
-	p := b.Config.Resources.QualityMonitors["my_monitor"]
-	assert.Equal(t, "main.test.prod", p.TableName)
-	assert.Equal(t, "/Shared/provider-test/databricks_monitoring/main.test.thing1", p.AssetsDir)
-	assert.Equal(t, "main.prod", p.OutputSchemaName)
-
-	inferenceLog := p.InferenceLog
-	assert.Equal(t, []string{"1 day", "1 hour"}, inferenceLog.Granularities)
-	assert.Equal(t, "timestamp_prod", p.InferenceLog.TimestampCol)
-	assert.Equal(t, "prediction_prod", p.InferenceLog.PredictionCol)
-	assert.Equal(t, "model_id_prod", p.InferenceLog.ModelIdCol)
-	assert.Equal(t, catalog.MonitorInferenceLogProblemType("PROBLEM_TYPE_REGRESSION"), p.InferenceLog.ProblemType)
-}

From 41bbd89257285707b3c3df9b9e5b92d6bcf8f1d1 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 20 Jan 2025 18:21:34 +0100
Subject: [PATCH 117/247] Clean up unnecessary cleanup of inferred flag (#2193)

## Changes
The SelectTarget mutator (part of Load phase) clears bundle.git.inferred
flag but it is not set until later - Initialize phase / LoadGitDetails
mutator.

## Tests
Existing tests.
---
 bundle/config/root.go | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/bundle/config/root.go b/bundle/config/root.go
index 21804110a..b974bcec5 100644
--- a/bundle/config/root.go
+++ b/bundle/config/root.go
@@ -388,14 +388,6 @@ func (r *Root) MergeTargetOverrides(name string) error {
 			return err
 		}
 
-		// If the branch was overridden, we need to clear the inferred flag.
-		if branch := v.Get("branch"); branch.Kind() != dyn.KindInvalid {
-			out, err = dyn.SetByPath(out, dyn.NewPath(dyn.Key("inferred")), dyn.V(false))
-			if err != nil {
-				return err
-			}
-		}
-
 		// Set the merged value.
 		root, err = dyn.SetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("git")), out)
 		if err != nil {

From 33613b5d2a5ff036f1d799637ffa9f222523e323 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 21 Jan 2025 12:27:02 +0100
Subject: [PATCH 118/247] Add test for #2181 /Workspace not prepended (#2188)

---
 .../prepend-workspace-var/databricks.yml      | 24 +++++++
 .../prepend-workspace-var/output.txt          | 67 +++++++++++++++++++
 .../variables/prepend-workspace-var/script    |  2 +
 3 files changed, 93 insertions(+)
 create mode 100644 acceptance/bundle/variables/prepend-workspace-var/databricks.yml
 create mode 100644 acceptance/bundle/variables/prepend-workspace-var/output.txt
 create mode 100644 acceptance/bundle/variables/prepend-workspace-var/script

diff --git a/acceptance/bundle/variables/prepend-workspace-var/databricks.yml b/acceptance/bundle/variables/prepend-workspace-var/databricks.yml
new file mode 100644
index 000000000..c843752f8
--- /dev/null
+++ b/acceptance/bundle/variables/prepend-workspace-var/databricks.yml
@@ -0,0 +1,24 @@
+workspace:
+  profile: profile_name
+  root_path: ${var.workspace_root}/path/to/root
+
+variables:
+  workspace_root:
+    description: "root directory in the Databricks workspace to store the asset bundle and associated artifacts"
+    default: /Users/${workspace.current_user.userName}
+
+targets:
+  dev:
+    default: true
+  prod:
+    variables:
+      workspace_root: /Shared
+
+resources:
+  jobs:
+    my_job:
+      tasks:
+        - existing_cluster_id: 500
+          python_wheel_task:
+            named_parameters:
+              conf-file: "${workspace.file_path}/path/to/config.yaml"
diff --git a/acceptance/bundle/variables/prepend-workspace-var/output.txt b/acceptance/bundle/variables/prepend-workspace-var/output.txt
new file mode 100644
index 000000000..575fac6d4
--- /dev/null
+++ b/acceptance/bundle/variables/prepend-workspace-var/output.txt
@@ -0,0 +1,67 @@
+/Workspace should be prepended on all paths, but it is not the case:
+{
+  "bundle": {
+    "environment": "dev",
+    "git": {
+      "bundle_root_path": ".",
+      "inferred": true
+    },
+    "target": "dev",
+    "terraform": {
+      "exec_path": "$TMPHOME"
+    }
+  },
+  "resources": {
+    "jobs": {
+      "my_job": {
+        "deployment": {
+          "kind": "BUNDLE",
+          "metadata_file_path": "/Users/$USERNAME/path/to/root/state/metadata.json"
+        },
+        "edit_mode": "UI_LOCKED",
+        "format": "MULTI_TASK",
+        "permissions": [],
+        "queue": {
+          "enabled": true
+        },
+        "tags": {},
+        "tasks": [
+          {
+            "existing_cluster_id": "500",
+            "python_wheel_task": {
+              "named_parameters": {
+                "conf-file": "/Users/$USERNAME/path/to/root/files/path/to/config.yaml"
+              }
+            },
+            "task_key": ""
+          }
+        ]
+      }
+    }
+  },
+  "sync": {
+    "paths": [
+      "."
+    ]
+  },
+  "targets": null,
+  "variables": {
+    "workspace_root": {
+      "default": "/Users/$USERNAME",
+      "description": "root directory in the Databricks workspace to store the asset bundle and associated artifacts",
+      "value": "/Users/$USERNAME"
+    }
+  },
+  "workspace": {
+    "artifact_path": "/Users/$USERNAME/path/to/root/artifacts",
+    "current_user": {
+      "short_name": "$USERNAME",
+      "userName": "$USERNAME"
+    },
+    "file_path": "/Users/$USERNAME/path/to/root/files",
+    "profile": "profile_name",
+    "resource_path": "/Users/$USERNAME/path/to/root/resources",
+    "root_path": "/Users/$USERNAME/path/to/root",
+    "state_path": "/Users/$USERNAME/path/to/root/state"
+  }
+}
\ No newline at end of file
diff --git a/acceptance/bundle/variables/prepend-workspace-var/script b/acceptance/bundle/variables/prepend-workspace-var/script
new file mode 100644
index 000000000..de6bc8a17
--- /dev/null
+++ b/acceptance/bundle/variables/prepend-workspace-var/script
@@ -0,0 +1,2 @@
+echo /Workspace should be prepended on all paths, but it is not the case: #2181
+$CLI bundle validate -o json

From de5155ed0a4635d2233b8e74544e1401a01eb786 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 21 Jan 2025 12:50:28 +0100
Subject: [PATCH 119/247] Add acceptance for test for sync.paths equal to two
 dots (#2196)

Based on integration test from @andrewnester in #2194

Manually checked that this databricks.yml passes validation on v0.235.0
but fails on v0.236.0, very like it was broken in
https://github.com/databricks/cli/pull/1945

This also adds replacements for tmpdir, it's parent and (just in case)
grand parent.
---
 acceptance/acceptance_test.go                      |  7 +++++++
 acceptance/bundle/sync-paths-dotdot/databricks.yml |  5 +++++
 acceptance/bundle/sync-paths-dotdot/output.txt     | 11 +++++++++++
 acceptance/bundle/sync-paths-dotdot/script         |  1 +
 4 files changed, 24 insertions(+)
 create mode 100644 acceptance/bundle/sync-paths-dotdot/databricks.yml
 create mode 100644 acceptance/bundle/sync-paths-dotdot/output.txt
 create mode 100644 acceptance/bundle/sync-paths-dotdot/script

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 5f1181313..850d3bf9d 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -137,6 +137,13 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		tmpDir = t.TempDir()
 	}
 
+	repls.Set("/private"+tmpDir, "$TMPDIR")
+	repls.Set("/private"+filepath.Dir(tmpDir), "$TMPPARENT")
+	repls.Set("/private"+filepath.Dir(filepath.Dir(tmpDir)), "$TMPGPARENT")
+	repls.Set(tmpDir, "$TMPDIR")
+	repls.Set(filepath.Dir(tmpDir), "$TMPPARENT")
+	repls.Set(filepath.Dir(filepath.Dir(tmpDir)), "$TMPGPARENT")
+
 	scriptContents := readMergedScriptContents(t, dir)
 	testutil.WriteFile(t, filepath.Join(tmpDir, EntryPointScript), scriptContents)
 
diff --git a/acceptance/bundle/sync-paths-dotdot/databricks.yml b/acceptance/bundle/sync-paths-dotdot/databricks.yml
new file mode 100644
index 000000000..7215ffea2
--- /dev/null
+++ b/acceptance/bundle/sync-paths-dotdot/databricks.yml
@@ -0,0 +1,5 @@
+bundle:
+  name: test-bundle
+sync:
+  paths:
+    - ..
diff --git a/acceptance/bundle/sync-paths-dotdot/output.txt b/acceptance/bundle/sync-paths-dotdot/output.txt
new file mode 100644
index 000000000..11db3e9ee
--- /dev/null
+++ b/acceptance/bundle/sync-paths-dotdot/output.txt
@@ -0,0 +1,11 @@
+Error: path "$TMPPARENT" is not within repository root "$TMPDIR"
+
+Name: test-bundle
+Target: default
+Workspace:
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/test-bundle/default
+
+Found 1 error
+
+Exit code: 1
diff --git a/acceptance/bundle/sync-paths-dotdot/script b/acceptance/bundle/sync-paths-dotdot/script
new file mode 100644
index 000000000..72555b332
--- /dev/null
+++ b/acceptance/bundle/sync-paths-dotdot/script
@@ -0,0 +1 @@
+$CLI bundle validate

From 34a37cf4a8d7248bce2c2291018c6d02c217c653 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 21 Jan 2025 13:47:34 +0100
Subject: [PATCH 120/247] Clone ReplacementContext before passing into test
 (#2198)

## Changes
- Add a new method Clone() on ReplacementContext
- Use it when passing common replacements to test cases.

## Tests
Manually. I have a different branch where this bug manifested and this
change helped.
---
 acceptance/acceptance_test.go | 2 +-
 libs/testdiff/replacement.go  | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 850d3bf9d..12fe6536f 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -99,7 +99,7 @@ func TestAccept(t *testing.T) {
 		testName := strings.ReplaceAll(dir, "\\", "/")
 		t.Run(testName, func(t *testing.T) {
 			t.Parallel()
-			runTest(t, dir, coverDir, repls)
+			runTest(t, dir, coverDir, repls.Clone())
 		})
 	}
 }
diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go
index 1ab976109..207f425aa 100644
--- a/libs/testdiff/replacement.go
+++ b/libs/testdiff/replacement.go
@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"regexp"
+	"slices"
 	"strings"
 
 	"github.com/databricks/cli/internal/testutil"
@@ -31,6 +32,10 @@ type ReplacementsContext struct {
 	Repls []Replacement
 }
 
+func (r *ReplacementsContext) Clone() ReplacementsContext {
+	return ReplacementsContext{Repls: slices.Clone(r.Repls)}
+}
+
 func (r *ReplacementsContext) Replace(s string) string {
 	// QQQ Should probably only replace whole words
 	for _, repl := range r.Repls {

From 3a32c63919d06408d37557b651b6d55e4fc71247 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 21 Jan 2025 22:21:12 +0100
Subject: [PATCH 121/247] Add -inprocess mode for acceptance tests (#2184)

## Changes
- If you pass -inprocess flag to acceptance tests, they will run in the
same process as test itself. This enables debugging.
- If you set singleTest variable on top of acceptance_test.go, you'll
only run that test and with inprocess mode. This is intended for
debugging in VSCode.
- (minor) Converted KeepTmp to flag -keeptmp from env var KEEP_TMP for
consistency with other flags.

## Tests
- I verified that acceptance tests pass with -inprocess mode: `go test
-inprocess < /dev/null | cat`
- I verified that debugging in VSCode works: set a test name in
singleTest variable, set breakpoints inside CLI and click "debug test"
in VSCode.
---
 acceptance/acceptance_test.go | 65 +++++++++++++++++++++++++++----
 acceptance/bin/callserver.py  | 31 +++++++++++++++
 acceptance/cmd_server_test.go | 73 +++++++++++++++++++++++++++++++++++
 3 files changed, 162 insertions(+), 7 deletions(-)
 create mode 100755 acceptance/bin/callserver.py
 create mode 100644 acceptance/cmd_server_test.go

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 12fe6536f..cfcb0d29f 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -3,6 +3,7 @@ package acceptance_test
 import (
 	"context"
 	"errors"
+	"flag"
 	"fmt"
 	"io"
 	"os"
@@ -23,7 +24,22 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-var KeepTmp = os.Getenv("KEEP_TMP") != ""
+var KeepTmp bool
+
+// In order to debug CLI running under acceptance test, set this to full subtest name, e.g. "bundle/variables/empty"
+// Then install your breakpoints and click "debug test" near TestAccept in VSCODE.
+// example: var singleTest = "bundle/variables/empty"
+var singleTest = ""
+
+// If enabled, instead of compiling and running CLI externally, we'll start in-process server that accepts and runs
+// CLI commands. The $CLI in test scripts is a helper that just forwards command-line arguments to this server (see bin/callserver.py).
+// Also disables parallelism in tests.
+var InprocessMode bool
+
+func init() {
+	flag.BoolVar(&InprocessMode, "inprocess", singleTest != "", "Run CLI in the same process as test (for debugging)")
+	flag.BoolVar(&KeepTmp, "keeptmp", false, "Do not delete TMP directory after run")
+}
 
 const (
 	EntryPointScript = "script"
@@ -38,6 +54,23 @@ var Scripts = map[string]bool{
 }
 
 func TestAccept(t *testing.T) {
+	testAccept(t, InprocessMode, "")
+}
+
+func TestInprocessMode(t *testing.T) {
+	if InprocessMode {
+		t.Skip("Already tested by TestAccept")
+	}
+	if runtime.GOOS == "windows" {
+		// -  catalogs                               A catalog is the first layer of Unity Catalog’s three-level namespace.
+		// +  catalogs                               A catalog is the first layer of Unity Catalog�s three-level namespace.
+		t.Skip("Fails on CI on unicode characters")
+	}
+	require.NotZero(t, testAccept(t, true, "help"))
+}
+
+func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
+	repls := testdiff.ReplacementsContext{}
 	cwd, err := os.Getwd()
 	require.NoError(t, err)
 
@@ -50,16 +83,22 @@ func TestAccept(t *testing.T) {
 		t.Logf("Writing coverage to %s", coverDir)
 	}
 
-	execPath := BuildCLI(t, cwd, coverDir)
-	// $CLI is what test scripts are using
+	execPath := ""
+
+	if InprocessMode {
+		cmdServer := StartCmdServer(t)
+		t.Setenv("CMD_SERVER_URL", cmdServer.URL)
+		execPath = filepath.Join(cwd, "bin", "callserver.py")
+	} else {
+		execPath = BuildCLI(t, cwd, coverDir)
+	}
+
 	t.Setenv("CLI", execPath)
+	repls.Set(execPath, "$CLI")
 
 	// Make helper scripts available
 	t.Setenv("PATH", fmt.Sprintf("%s%c%s", filepath.Join(cwd, "bin"), os.PathListSeparator, os.Getenv("PATH")))
 
-	repls := testdiff.ReplacementsContext{}
-	repls.Set(execPath, "$CLI")
-
 	tempHomeDir := t.TempDir()
 	repls.Set(tempHomeDir, "$TMPHOME")
 	t.Logf("$TMPHOME=%v", tempHomeDir)
@@ -95,13 +134,25 @@ func TestAccept(t *testing.T) {
 	testDirs := getTests(t)
 	require.NotEmpty(t, testDirs)
 
+	if singleTest != "" {
+		testDirs = slices.DeleteFunc(testDirs, func(n string) bool {
+			return n != singleTest
+		})
+		require.NotEmpty(t, testDirs, "singleTest=%#v did not match any tests\n%#v", singleTest, testDirs)
+	}
+
 	for _, dir := range testDirs {
 		testName := strings.ReplaceAll(dir, "\\", "/")
 		t.Run(testName, func(t *testing.T) {
-			t.Parallel()
+			if !InprocessMode {
+				t.Parallel()
+			}
+
 			runTest(t, dir, coverDir, repls.Clone())
 		})
 	}
+
+	return len(testDirs)
 }
 
 func getTests(t *testing.T) []string {
diff --git a/acceptance/bin/callserver.py b/acceptance/bin/callserver.py
new file mode 100755
index 000000000..294ef8fdb
--- /dev/null
+++ b/acceptance/bin/callserver.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import sys
+import os
+import json
+import urllib.request
+from urllib.parse import urlencode
+
+env = {}
+for key, value in os.environ.items():
+    if len(value) > 10_000:
+        sys.stderr.write(f"Dropping key={key} value len={len(value)}\n")
+        continue
+    env[key] = value
+
+q = {
+    "args": " ".join(sys.argv[1:]),
+    "cwd": os.getcwd(),
+    "env": json.dumps(env),
+}
+
+url = os.environ["CMD_SERVER_URL"] + "/?" + urlencode(q)
+if len(url) > 100_000:
+    sys.exit("url too large")
+
+resp = urllib.request.urlopen(url)
+assert resp.status == 200, (resp.status, resp.url, resp.headers)
+result = json.load(resp)
+sys.stderr.write(result["stderr"])
+sys.stdout.write(result["stdout"])
+exitcode = int(result["exitcode"])
+sys.exit(exitcode)
diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go
new file mode 100644
index 000000000..28feec1bd
--- /dev/null
+++ b/acceptance/cmd_server_test.go
@@ -0,0 +1,73 @@
+package acceptance_test
+
+import (
+	"encoding/json"
+	"net/http"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/databricks/cli/internal/testcli"
+	"github.com/stretchr/testify/require"
+)
+
+func StartCmdServer(t *testing.T) *TestServer {
+	server := StartServer(t)
+	server.Handle("/", func(r *http.Request) (any, error) {
+		q := r.URL.Query()
+		args := strings.Split(q.Get("args"), " ")
+
+		var env map[string]string
+		require.NoError(t, json.Unmarshal([]byte(q.Get("env")), &env))
+
+		for key, val := range env {
+			defer Setenv(t, key, val)()
+		}
+
+		defer Chdir(t, q.Get("cwd"))()
+
+		c := testcli.NewRunner(t, r.Context(), args...)
+		c.Verbose = false
+		stdout, stderr, err := c.Run()
+		result := map[string]any{
+			"stdout": stdout.String(),
+			"stderr": stderr.String(),
+		}
+		exitcode := 0
+		if err != nil {
+			exitcode = 1
+		}
+		result["exitcode"] = exitcode
+		return result, nil
+	})
+	return server
+}
+
+// Chdir variant that is intended to be used with defer so that it can switch back before function ends.
+// This is unlike testutil.Chdir which switches back only when tests end.
+func Chdir(t *testing.T, cwd string) func() {
+	require.NotEmpty(t, cwd)
+	prevDir, err := os.Getwd()
+	require.NoError(t, err)
+	err = os.Chdir(cwd)
+	require.NoError(t, err)
+	return func() {
+		_ = os.Chdir(prevDir)
+	}
+}
+
+// Setenv variant that is intended to be used with defer so that it can switch back before function ends.
+// This is unlike t.Setenv which switches back only when tests end.
+func Setenv(t *testing.T, key, value string) func() {
+	prevVal, exists := os.LookupEnv(key)
+
+	require.NoError(t, os.Setenv(key, value))
+
+	return func() {
+		if exists {
+			_ = os.Setenv(key, prevVal)
+		} else {
+			_ = os.Unsetenv(key)
+		}
+	}
+}

From fde30ff1ab5bf504e326f58adfd9f429d88b3b6d Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 22 Jan 2025 11:17:45 +0100
Subject: [PATCH 122/247] Add a test for sync root outside of git root (#2202)

- Move acceptance/bundle/sync-paths-dotdot test to
acceptance/bundle/syncroot/dotdot-notgit
- Add new test acceptance/bundle/syncroot/dotdot-git

Fix replacer to work with this test and on Windows:
- Make PATH work on Windows by using EvalSymlinks.
- Make concatenated path match within JSON but stripping quotes.
---
 acceptance/acceptance_test.go                 | 12 +++---
 acceptance/bundle/sync-paths-dotdot/script    |  1 -
 .../dotdot-git}/databricks.yml                |  0
 .../dotdot-git}/output.txt                    |  2 +-
 acceptance/bundle/syncroot/dotdot-git/script  |  6 +++
 .../syncroot/dotdot-nogit/databricks.yml      |  5 +++
 .../bundle/syncroot/dotdot-nogit/output.txt   | 11 ++++++
 .../bundle/syncroot/dotdot-nogit/script       |  2 +
 libs/testdiff/replacement.go                  | 39 ++++++++++++++++++-
 9 files changed, 69 insertions(+), 9 deletions(-)
 delete mode 100644 acceptance/bundle/sync-paths-dotdot/script
 rename acceptance/bundle/{sync-paths-dotdot => syncroot/dotdot-git}/databricks.yml (100%)
 rename acceptance/bundle/{sync-paths-dotdot => syncroot/dotdot-git}/output.txt (69%)
 create mode 100644 acceptance/bundle/syncroot/dotdot-git/script
 create mode 100644 acceptance/bundle/syncroot/dotdot-nogit/databricks.yml
 create mode 100644 acceptance/bundle/syncroot/dotdot-nogit/output.txt
 create mode 100644 acceptance/bundle/syncroot/dotdot-nogit/script

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index cfcb0d29f..9a4564ffa 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -188,12 +188,12 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		tmpDir = t.TempDir()
 	}
 
-	repls.Set("/private"+tmpDir, "$TMPDIR")
-	repls.Set("/private"+filepath.Dir(tmpDir), "$TMPPARENT")
-	repls.Set("/private"+filepath.Dir(filepath.Dir(tmpDir)), "$TMPGPARENT")
-	repls.Set(tmpDir, "$TMPDIR")
-	repls.Set(filepath.Dir(tmpDir), "$TMPPARENT")
-	repls.Set(filepath.Dir(filepath.Dir(tmpDir)), "$TMPGPARENT")
+	// Converts C:\Users\DENIS~1.BIL -> C:\Users\denis.bilenko
+	tmpDirEvalled, err1 := filepath.EvalSymlinks(tmpDir)
+	if err1 == nil && tmpDirEvalled != tmpDir {
+		repls.SetPathWithParents(tmpDirEvalled, "$TMPDIR")
+	}
+	repls.SetPathWithParents(tmpDir, "$TMPDIR")
 
 	scriptContents := readMergedScriptContents(t, dir)
 	testutil.WriteFile(t, filepath.Join(tmpDir, EntryPointScript), scriptContents)
diff --git a/acceptance/bundle/sync-paths-dotdot/script b/acceptance/bundle/sync-paths-dotdot/script
deleted file mode 100644
index 72555b332..000000000
--- a/acceptance/bundle/sync-paths-dotdot/script
+++ /dev/null
@@ -1 +0,0 @@
-$CLI bundle validate
diff --git a/acceptance/bundle/sync-paths-dotdot/databricks.yml b/acceptance/bundle/syncroot/dotdot-git/databricks.yml
similarity index 100%
rename from acceptance/bundle/sync-paths-dotdot/databricks.yml
rename to acceptance/bundle/syncroot/dotdot-git/databricks.yml
diff --git a/acceptance/bundle/sync-paths-dotdot/output.txt b/acceptance/bundle/syncroot/dotdot-git/output.txt
similarity index 69%
rename from acceptance/bundle/sync-paths-dotdot/output.txt
rename to acceptance/bundle/syncroot/dotdot-git/output.txt
index 11db3e9ee..f1dc5fb01 100644
--- a/acceptance/bundle/sync-paths-dotdot/output.txt
+++ b/acceptance/bundle/syncroot/dotdot-git/output.txt
@@ -1,4 +1,4 @@
-Error: path "$TMPPARENT" is not within repository root "$TMPDIR"
+Error: path "$TMPDIR" is not within repository root "$TMPDIR/myrepo"
 
 Name: test-bundle
 Target: default
diff --git a/acceptance/bundle/syncroot/dotdot-git/script b/acceptance/bundle/syncroot/dotdot-git/script
new file mode 100644
index 000000000..0706a1d5e
--- /dev/null
+++ b/acceptance/bundle/syncroot/dotdot-git/script
@@ -0,0 +1,6 @@
+# This should error, we do not allow syncroot outside of git repo.
+mkdir myrepo
+cd myrepo
+cp ../databricks.yml .
+git-repo-init
+$CLI bundle validate | sed 's/\\\\/\//g'
diff --git a/acceptance/bundle/syncroot/dotdot-nogit/databricks.yml b/acceptance/bundle/syncroot/dotdot-nogit/databricks.yml
new file mode 100644
index 000000000..7215ffea2
--- /dev/null
+++ b/acceptance/bundle/syncroot/dotdot-nogit/databricks.yml
@@ -0,0 +1,5 @@
+bundle:
+  name: test-bundle
+sync:
+  paths:
+    - ..
diff --git a/acceptance/bundle/syncroot/dotdot-nogit/output.txt b/acceptance/bundle/syncroot/dotdot-nogit/output.txt
new file mode 100644
index 000000000..34059e276
--- /dev/null
+++ b/acceptance/bundle/syncroot/dotdot-nogit/output.txt
@@ -0,0 +1,11 @@
+Error: path "$TMPDIR_PARENT" is not within repository root "$TMPDIR"
+
+Name: test-bundle
+Target: default
+Workspace:
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/test-bundle/default
+
+Found 1 error
+
+Exit code: 1
diff --git a/acceptance/bundle/syncroot/dotdot-nogit/script b/acceptance/bundle/syncroot/dotdot-nogit/script
new file mode 100644
index 000000000..d3388903e
--- /dev/null
+++ b/acceptance/bundle/syncroot/dotdot-nogit/script
@@ -0,0 +1,2 @@
+# This should not error, syncroot can be outside bundle root.
+$CLI bundle validate
diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go
index 207f425aa..ca76b159c 100644
--- a/libs/testdiff/replacement.go
+++ b/libs/testdiff/replacement.go
@@ -3,7 +3,9 @@ package testdiff
 import (
 	"encoding/json"
 	"fmt"
+	"path/filepath"
 	"regexp"
+	"runtime"
 	"slices"
 	"strings"
 
@@ -74,13 +76,48 @@ func (r *ReplacementsContext) Set(old, new string) {
 	if err == nil {
 		encodedOld, err := json.Marshal(old)
 		if err == nil {
-			r.appendLiteral(string(encodedOld), string(encodedNew))
+			r.appendLiteral(trimQuotes(string(encodedOld)), trimQuotes(string(encodedNew)))
 		}
 	}
 
 	r.appendLiteral(old, new)
 }
 
+func trimQuotes(s string) string {
+	if len(s) > 0 && s[0] == '"' {
+		s = s[1:]
+	}
+	if len(s) > 0 && s[len(s)-1] == '"' {
+		s = s[:len(s)-1]
+	}
+	return s
+}
+
+func (r *ReplacementsContext) SetPath(old, new string) {
+	r.Set(old, new)
+
+	if runtime.GOOS != "windows" {
+		return
+	}
+
+	// Support both forward and backward slashes
+	m1 := strings.ReplaceAll(old, "\\", "/")
+	if m1 != old {
+		r.Set(m1, new)
+	}
+
+	m2 := strings.ReplaceAll(old, "/", "\\")
+	if m2 != old && m2 != m1 {
+		r.Set(m2, new)
+	}
+}
+
+func (r *ReplacementsContext) SetPathWithParents(old, new string) {
+	r.SetPath(old, new)
+	r.SetPath(filepath.Dir(old), new+"_PARENT")
+	r.SetPath(filepath.Dir(filepath.Dir(old)), new+"_GPARENT")
+}
+
 func PrepareReplacementsWorkspaceClient(t testutil.TestingT, r *ReplacementsContext, w *databricks.WorkspaceClient) {
 	t.Helper()
 	// in some clouds (gcp) w.Config.Host includes "https://" prefix in others it's really just a host (azure)

From c224be5c1f9c0d082c6453031f740158ffaab91c Mon Sep 17 00:00:00 2001
From: Ilya Kuznetsov <ilya.kuznetsov@databricks.com>
Date: Wed, 22 Jan 2025 11:30:17 +0100
Subject: [PATCH 123/247] Allow using variables in enum fields (#2199)

## Changes

It is possible to pass variable to enum fields but json-schema doesn't
accept it. This PR adds `oneOf` for enum types that includes `${var-*}`
pattern

## Tests

Manually checked in VSCode
---
 bundle/internal/schema/main.go               |  13 +
 bundle/internal/schema/testdata/pass/job.yml |   4 +
 bundle/schema/embed_test.go                  |   8 +-
 bundle/schema/jsonschema.json                | 916 +++++++++++++------
 4 files changed, 663 insertions(+), 278 deletions(-)

diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go
index 77927a966..39b859656 100644
--- a/bundle/internal/schema/main.go
+++ b/bundle/internal/schema/main.go
@@ -40,6 +40,19 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema.
 		}
 	}
 
+	// Allows using variables in enum fields
+	if s.Type == jsonschema.StringType && s.Enum != nil {
+		return jsonschema.Schema{
+			OneOf: []jsonschema.Schema{
+				s,
+				{
+					Type:    jsonschema.StringType,
+					Pattern: interpolationPattern("var"),
+				},
+			},
+		}
+	}
+
 	switch s.Type {
 	case jsonschema.ArrayType, jsonschema.ObjectType:
 		// arrays and objects can have complex variable values specified.
diff --git a/bundle/internal/schema/testdata/pass/job.yml b/bundle/internal/schema/testdata/pass/job.yml
index e13a52c03..ec447ba39 100644
--- a/bundle/internal/schema/testdata/pass/job.yml
+++ b/bundle/internal/schema/testdata/pass/job.yml
@@ -13,6 +13,8 @@ variables:
   simplevar:
     default: true
     description: "simplevar description"
+  schedule_status:
+    default: "PAUSED"
 
   complexvar:
     default:
@@ -42,6 +44,8 @@ resources:
             dependencies:
               - python=3.7
             client: "myclient"
+      trigger:
+        pause_status: ${var.schedule_status}
       tags:
         foo: bar
         bar: baz
diff --git a/bundle/schema/embed_test.go b/bundle/schema/embed_test.go
index 59f1458cb..03d2165e4 100644
--- a/bundle/schema/embed_test.go
+++ b/bundle/schema/embed_test.go
@@ -59,8 +59,8 @@ func TestJsonSchema(t *testing.T) {
 	}
 
 	providers := walk(s.Definitions, "github.com", "databricks", "databricks-sdk-go", "service", "jobs.GitProvider")
-	assert.Contains(t, providers.Enum, "gitHub")
-	assert.Contains(t, providers.Enum, "bitbucketCloud")
-	assert.Contains(t, providers.Enum, "gitHubEnterprise")
-	assert.Contains(t, providers.Enum, "bitbucketServer")
+	assert.Contains(t, providers.OneOf[0].Enum, "gitHub")
+	assert.Contains(t, providers.OneOf[0].Enum, "bitbucketCloud")
+	assert.Contains(t, providers.OneOf[0].Enum, "gitHubEnterprise")
+	assert.Contains(t, providers.OneOf[0].Enum, "bitbucketServer")
 }
diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json
index 57b0cd231..b3158792c 100644
--- a/bundle/schema/jsonschema.json
+++ b/bundle/schema/jsonschema.json
@@ -1663,19 +1663,35 @@
               ]
             },
             "apps.AppDeploymentMode": {
-              "type": "string",
-              "enum": [
-                "SNAPSHOT",
-                "AUTO_SYNC"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "enum": [
+                    "SNAPSHOT",
+                    "AUTO_SYNC"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "apps.AppDeploymentState": {
-              "type": "string",
-              "enum": [
-                "SUCCEEDED",
-                "FAILED",
-                "IN_PROGRESS",
-                "CANCELLED"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "enum": [
+                    "SUCCEEDED",
+                    "FAILED",
+                    "IN_PROGRESS",
+                    "CANCELLED"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "apps.AppDeploymentStatus": {
@@ -1758,12 +1774,20 @@
               ]
             },
             "apps.AppResourceJobJobPermission": {
-              "type": "string",
-              "enum": [
-                "CAN_MANAGE",
-                "IS_OWNER",
-                "CAN_MANAGE_RUN",
-                "CAN_VIEW"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "enum": [
+                    "CAN_MANAGE",
+                    "IS_OWNER",
+                    "CAN_MANAGE_RUN",
+                    "CAN_VIEW"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "apps.AppResourceSecret": {
@@ -1795,12 +1819,20 @@
               ]
             },
             "apps.AppResourceSecretSecretPermission": {
-              "type": "string",
-              "description": "Permission to grant on the secret scope. Supported permissions are: \"READ\", \"WRITE\", \"MANAGE\".",
-              "enum": [
-                "READ",
-                "WRITE",
-                "MANAGE"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Permission to grant on the secret scope. Supported permissions are: \"READ\", \"WRITE\", \"MANAGE\".",
+                  "enum": [
+                    "READ",
+                    "WRITE",
+                    "MANAGE"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "apps.AppResourceServingEndpoint": {
@@ -1828,11 +1860,19 @@
               ]
             },
             "apps.AppResourceServingEndpointServingEndpointPermission": {
-              "type": "string",
-              "enum": [
-                "CAN_MANAGE",
-                "CAN_QUERY",
-                "CAN_VIEW"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "enum": [
+                    "CAN_MANAGE",
+                    "CAN_QUERY",
+                    "CAN_VIEW"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "apps.AppResourceSqlWarehouse": {
@@ -1860,20 +1900,36 @@
               ]
             },
             "apps.AppResourceSqlWarehouseSqlWarehousePermission": {
-              "type": "string",
-              "enum": [
-                "CAN_MANAGE",
-                "CAN_USE",
-                "IS_OWNER"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "enum": [
+                    "CAN_MANAGE",
+                    "CAN_USE",
+                    "IS_OWNER"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "apps.ApplicationState": {
-              "type": "string",
-              "enum": [
-                "DEPLOYING",
-                "RUNNING",
-                "CRASHED",
-                "UNAVAILABLE"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "enum": [
+                    "DEPLOYING",
+                    "RUNNING",
+                    "CRASHED",
+                    "UNAVAILABLE"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "apps.ApplicationStatus": {
@@ -1897,15 +1953,23 @@
               ]
             },
             "apps.ComputeState": {
-              "type": "string",
-              "enum": [
-                "ERROR",
-                "DELETING",
-                "STARTING",
-                "STOPPING",
-                "UPDATING",
-                "STOPPED",
-                "ACTIVE"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "enum": [
+                    "ERROR",
+                    "DELETING",
+                    "STARTING",
+                    "STOPPING",
+                    "UPDATING",
+                    "STOPPED",
+                    "ACTIVE"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "apps.ComputeStatus": {
@@ -1959,11 +2023,19 @@
               ]
             },
             "catalog.MonitorCronSchedulePauseStatus": {
-              "type": "string",
-              "description": "Read only field that indicates whether a schedule is paused or not.",
-              "enum": [
-                "UNPAUSED",
-                "PAUSED"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Read only field that indicates whether a schedule is paused or not.",
+                  "enum": [
+                    "UNPAUSED",
+                    "PAUSED"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "catalog.MonitorDataClassificationConfig": {
@@ -2052,11 +2124,19 @@
               ]
             },
             "catalog.MonitorInferenceLogProblemType": {
-              "type": "string",
-              "description": "Problem type the model aims to solve. Determines the type of model-quality metrics that will be computed.",
-              "enum": [
-                "PROBLEM_TYPE_CLASSIFICATION",
-                "PROBLEM_TYPE_REGRESSION"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Problem type the model aims to solve. Determines the type of model-quality metrics that will be computed.",
+                  "enum": [
+                    "PROBLEM_TYPE_CLASSIFICATION",
+                    "PROBLEM_TYPE_REGRESSION"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "catalog.MonitorMetric": {
@@ -2101,12 +2181,20 @@
               ]
             },
             "catalog.MonitorMetricType": {
-              "type": "string",
-              "description": "Can only be one of ``\"CUSTOM_METRIC_TYPE_AGGREGATE\"``, ``\"CUSTOM_METRIC_TYPE_DERIVED\"``, or ``\"CUSTOM_METRIC_TYPE_DRIFT\"``.\nThe ``\"CUSTOM_METRIC_TYPE_AGGREGATE\"`` and ``\"CUSTOM_METRIC_TYPE_DERIVED\"`` metrics\nare computed on a single table, whereas the ``\"CUSTOM_METRIC_TYPE_DRIFT\"`` compare metrics across\nbaseline and input table, or across the two consecutive time windows.\n- CUSTOM_METRIC_TYPE_AGGREGATE: only depend on the existing columns in your table\n- CUSTOM_METRIC_TYPE_DERIVED: depend on previously computed aggregate metrics\n- CUSTOM_METRIC_TYPE_DRIFT:  depend on previously computed aggregate or derived metrics\n",
-              "enum": [
-                "CUSTOM_METRIC_TYPE_AGGREGATE",
-                "CUSTOM_METRIC_TYPE_DERIVED",
-                "CUSTOM_METRIC_TYPE_DRIFT"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Can only be one of ``\"CUSTOM_METRIC_TYPE_AGGREGATE\"``, ``\"CUSTOM_METRIC_TYPE_DERIVED\"``, or ``\"CUSTOM_METRIC_TYPE_DRIFT\"``.\nThe ``\"CUSTOM_METRIC_TYPE_AGGREGATE\"`` and ``\"CUSTOM_METRIC_TYPE_DERIVED\"`` metrics\nare computed on a single table, whereas the ``\"CUSTOM_METRIC_TYPE_DRIFT\"`` compare metrics across\nbaseline and input table, or across the two consecutive time windows.\n- CUSTOM_METRIC_TYPE_AGGREGATE: only depend on the existing columns in your table\n- CUSTOM_METRIC_TYPE_DERIVED: depend on previously computed aggregate metrics\n- CUSTOM_METRIC_TYPE_DRIFT:  depend on previously computed aggregate or derived metrics\n",
+                  "enum": [
+                    "CUSTOM_METRIC_TYPE_AGGREGATE",
+                    "CUSTOM_METRIC_TYPE_DERIVED",
+                    "CUSTOM_METRIC_TYPE_DRIFT"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "catalog.MonitorNotifications": {
@@ -2170,10 +2258,18 @@
               ]
             },
             "catalog.VolumeType": {
-              "type": "string",
-              "enum": [
-                "EXTERNAL",
-                "MANAGED"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "enum": [
+                    "EXTERNAL",
+                    "MANAGED"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "compute.Adlsgen2Info": {
@@ -2272,12 +2368,20 @@
               ]
             },
             "compute.AwsAvailability": {
-              "type": "string",
-              "description": "Availability type used for all subsequent nodes past the `first_on_demand` ones.\n\nNote: If `first_on_demand` is zero, this availability type will be used for the entire cluster.\n",
-              "enum": [
-                "SPOT",
-                "ON_DEMAND",
-                "SPOT_WITH_FALLBACK"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Availability type used for all subsequent nodes past the `first_on_demand` ones.\n\nNote: If `first_on_demand` is zero, this availability type will be used for the entire cluster.\n",
+                  "enum": [
+                    "SPOT",
+                    "ON_DEMAND",
+                    "SPOT_WITH_FALLBACK"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "compute.AzureAttributes": {
@@ -2310,12 +2414,20 @@
               ]
             },
             "compute.AzureAvailability": {
-              "type": "string",
-              "description": "Availability type used for all subsequent nodes past the `first_on_demand` ones.\nNote: If `first_on_demand` is zero (which only happens on pool clusters), this availability\ntype will be used for the entire cluster.",
-              "enum": [
-                "SPOT_AZURE",
-                "ON_DEMAND_AZURE",
-                "SPOT_WITH_FALLBACK_AZURE"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Availability type used for all subsequent nodes past the `first_on_demand` ones.\nNote: If `first_on_demand` is zero (which only happens on pool clusters), this availability\ntype will be used for the entire cluster.",
+                  "enum": [
+                    "SPOT_AZURE",
+                    "ON_DEMAND_AZURE",
+                    "SPOT_WITH_FALLBACK_AZURE"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "compute.ClientsTypes": {
@@ -2492,19 +2604,27 @@
               ]
             },
             "compute.DataSecurityMode": {
-              "type": "string",
-              "description": "Data security mode decides what data governance model to use when accessing data\nfrom a cluster.\n\nThe following modes can only be used with `kind`.\n* `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration.\n* `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`.\n* `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`.\n\nThe following modes can be used regardless of `kind`.\n* `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode.\n* `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode.\n* `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.\n\nThe following modes are deprecated starting with Databricks Runtime 15.0 and\nwill be removed for future Databricks Runtime versions:\n\n* `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters.\n* `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters.\n* `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters.\n* `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled.\n",
-              "enum": [
-                "DATA_SECURITY_MODE_AUTO",
-                "DATA_SECURITY_MODE_STANDARD",
-                "DATA_SECURITY_MODE_DEDICATED",
-                "NONE",
-                "SINGLE_USER",
-                "USER_ISOLATION",
-                "LEGACY_TABLE_ACL",
-                "LEGACY_PASSTHROUGH",
-                "LEGACY_SINGLE_USER",
-                "LEGACY_SINGLE_USER_STANDARD"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Data security mode decides what data governance model to use when accessing data\nfrom a cluster.\n\nThe following modes can only be used with `kind`.\n* `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration.\n* `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`.\n* `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`.\n\nThe following modes can be used regardless of `kind`.\n* `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode.\n* `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode.\n* `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.\n\nThe following modes are deprecated starting with Databricks Runtime 15.0 and\nwill be removed for future Databricks Runtime versions:\n\n* `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters.\n* `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters.\n* `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters.\n* `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled.\n",
+                  "enum": [
+                    "DATA_SECURITY_MODE_AUTO",
+                    "DATA_SECURITY_MODE_STANDARD",
+                    "DATA_SECURITY_MODE_DEDICATED",
+                    "NONE",
+                    "SINGLE_USER",
+                    "USER_ISOLATION",
+                    "LEGACY_TABLE_ACL",
+                    "LEGACY_PASSTHROUGH",
+                    "LEGACY_SINGLE_USER",
+                    "LEGACY_SINGLE_USER_STANDARD"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "compute.DbfsStorageInfo": {
@@ -2572,11 +2692,19 @@
               ]
             },
             "compute.EbsVolumeType": {
-              "type": "string",
-              "description": "The type of EBS volumes that will be launched with this cluster.",
-              "enum": [
-                "GENERAL_PURPOSE_SSD",
-                "THROUGHPUT_OPTIMIZED_HDD"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "The type of EBS volumes that will be launched with this cluster.",
+                  "enum": [
+                    "GENERAL_PURPOSE_SSD",
+                    "THROUGHPUT_OPTIMIZED_HDD"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "compute.Environment": {
@@ -2643,12 +2771,20 @@
               ]
             },
             "compute.GcpAvailability": {
-              "type": "string",
-              "description": "This field determines whether the instance pool will contain preemptible\nVMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable.",
-              "enum": [
-                "PREEMPTIBLE_GCP",
-                "ON_DEMAND_GCP",
-                "PREEMPTIBLE_WITH_FALLBACK_GCP"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "This field determines whether the instance pool will contain preemptible\nVMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable.",
+                  "enum": [
+                    "PREEMPTIBLE_GCP",
+                    "ON_DEMAND_GCP",
+                    "PREEMPTIBLE_WITH_FALLBACK_GCP"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "compute.GcsStorageInfo": {
@@ -2882,12 +3018,20 @@
               ]
             },
             "compute.RuntimeEngine": {
-              "type": "string",
-              "description": "Determines the cluster's runtime engine, either standard or Photon.\n\nThis field is not compatible with legacy `spark_version` values that contain `-photon-`.\nRemove `-photon-` from the `spark_version` and set `runtime_engine` to `PHOTON`.\n\nIf left unspecified, the runtime engine defaults to standard unless the spark_version\ncontains -photon-, in which case Photon will be used.\n",
-              "enum": [
-                "NULL",
-                "STANDARD",
-                "PHOTON"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Determines the cluster's runtime engine, either standard or Photon.\n\nThis field is not compatible with legacy `spark_version` values that contain `-photon-`.\nRemove `-photon-` from the `spark_version` and set `runtime_engine` to `PHOTON`.\n\nIf left unspecified, the runtime engine defaults to standard unless the spark_version\ncontains -photon-, in which case Photon will be used.\n",
+                  "enum": [
+                    "NULL",
+                    "STANDARD",
+                    "PHOTON"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "compute.S3StorageInfo": {
@@ -2999,10 +3143,18 @@
               ]
             },
             "dashboards.LifecycleState": {
-              "type": "string",
-              "enum": [
-                "ACTIVE",
-                "TRASHED"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "enum": [
+                    "ACTIVE",
+                    "TRASHED"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "jobs.CleanRoomsNotebookTask": {
@@ -3040,10 +3192,18 @@
               ]
             },
             "jobs.Condition": {
-              "type": "string",
-              "enum": [
-                "ANY_UPDATED",
-                "ALL_UPDATED"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "enum": [
+                    "ANY_UPDATED",
+                    "ALL_UPDATED"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "jobs.ConditionTask": {
@@ -3078,15 +3238,23 @@
               ]
             },
             "jobs.ConditionTaskOp": {
-              "type": "string",
-              "description": "* `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`.\n* `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” \u003e= “12”` will evaluate to `true`, `“10.0” \u003e= “12”` will evaluate to `false`.\n\nThe boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison.",
-              "enum": [
-                "EQUAL_TO",
-                "GREATER_THAN",
-                "GREATER_THAN_OR_EQUAL",
-                "LESS_THAN",
-                "LESS_THAN_OR_EQUAL",
-                "NOT_EQUAL"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "* `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`.\n* `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” \u003e= “12”` will evaluate to `true`, `“10.0” \u003e= “12”` will evaluate to `false`.\n\nThe boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison.",
+                  "enum": [
+                    "EQUAL_TO",
+                    "GREATER_THAN",
+                    "GREATER_THAN_OR_EQUAL",
+                    "LESS_THAN",
+                    "LESS_THAN_OR_EQUAL",
+                    "NOT_EQUAL"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "jobs.Continuous": {
@@ -3242,23 +3410,39 @@
               ]
             },
             "jobs.Format": {
-              "type": "string",
-              "enum": [
-                "SINGLE_TASK",
-                "MULTI_TASK"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "enum": [
+                    "SINGLE_TASK",
+                    "MULTI_TASK"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "jobs.GitProvider": {
-              "type": "string",
-              "enum": [
-                "gitHub",
-                "bitbucketCloud",
-                "azureDevOpsServices",
-                "gitHubEnterprise",
-                "bitbucketServer",
-                "gitLab",
-                "gitLabEnterpriseEdition",
-                "awsCodeCommit"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "enum": [
+                    "gitHub",
+                    "bitbucketCloud",
+                    "azureDevOpsServices",
+                    "gitHubEnterprise",
+                    "bitbucketServer",
+                    "gitLab",
+                    "gitLabEnterpriseEdition",
+                    "awsCodeCommit"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "jobs.GitSnapshot": {
@@ -3371,18 +3555,34 @@
               ]
             },
             "jobs.JobDeploymentKind": {
-              "type": "string",
-              "description": "* `BUNDLE`: The job is managed by Databricks Asset Bundle.",
-              "enum": [
-                "BUNDLE"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "* `BUNDLE`: The job is managed by Databricks Asset Bundle.",
+                  "enum": [
+                    "BUNDLE"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "jobs.JobEditMode": {
-              "type": "string",
-              "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified.",
-              "enum": [
-                "UI_LOCKED",
-                "EDITABLE"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified.",
+                  "enum": [
+                    "UI_LOCKED",
+                    "EDITABLE"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "jobs.JobEmailNotifications": {
@@ -3550,29 +3750,53 @@
               ]
             },
             "jobs.JobSourceDirtyState": {
-              "type": "string",
-              "description": "Dirty state indicates the job is not fully synced with the job specification\nin the remote repository.\n\nPossible values are:\n* `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced.\n* `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced.",
-              "enum": [
-                "NOT_SYNCED",
-                "DISCONNECTED"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Dirty state indicates the job is not fully synced with the job specification\nin the remote repository.\n\nPossible values are:\n* `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced.\n* `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced.",
+                  "enum": [
+                    "NOT_SYNCED",
+                    "DISCONNECTED"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "jobs.JobsHealthMetric": {
-              "type": "string",
-              "description": "Specifies the health metric that is being evaluated for a particular health rule.\n\n* `RUN_DURATION_SECONDS`: Expected total time for a run in seconds.\n* `STREAMING_BACKLOG_BYTES`: An estimate of the maximum bytes of data waiting to be consumed across all streams. This metric is in Public Preview.\n* `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag across all streams. This metric is in Public Preview.\n* `STREAMING_BACKLOG_SECONDS`: An estimate of the maximum consumer delay across all streams. This metric is in Public Preview.\n* `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all streams. This metric is in Public Preview.",
-              "enum": [
-                "RUN_DURATION_SECONDS",
-                "STREAMING_BACKLOG_BYTES",
-                "STREAMING_BACKLOG_RECORDS",
-                "STREAMING_BACKLOG_SECONDS",
-                "STREAMING_BACKLOG_FILES"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Specifies the health metric that is being evaluated for a particular health rule.\n\n* `RUN_DURATION_SECONDS`: Expected total time for a run in seconds.\n* `STREAMING_BACKLOG_BYTES`: An estimate of the maximum bytes of data waiting to be consumed across all streams. This metric is in Public Preview.\n* `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag across all streams. This metric is in Public Preview.\n* `STREAMING_BACKLOG_SECONDS`: An estimate of the maximum consumer delay across all streams. This metric is in Public Preview.\n* `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all streams. This metric is in Public Preview.",
+                  "enum": [
+                    "RUN_DURATION_SECONDS",
+                    "STREAMING_BACKLOG_BYTES",
+                    "STREAMING_BACKLOG_RECORDS",
+                    "STREAMING_BACKLOG_SECONDS",
+                    "STREAMING_BACKLOG_FILES"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "jobs.JobsHealthOperator": {
-              "type": "string",
-              "description": "Specifies the operator used to compare the health metric value with the specified threshold.",
-              "enum": [
-                "GREATER_THAN"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Specifies the operator used to compare the health metric value with the specified threshold.",
+                  "enum": [
+                    "GREATER_THAN"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "jobs.JobsHealthRule": {
@@ -3656,10 +3880,18 @@
               ]
             },
             "jobs.PauseStatus": {
-              "type": "string",
-              "enum": [
-                "UNPAUSED",
-                "PAUSED"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "enum": [
+                    "UNPAUSED",
+                    "PAUSED"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "jobs.PeriodicTriggerConfiguration": {
@@ -3689,11 +3921,19 @@
               ]
             },
             "jobs.PeriodicTriggerConfigurationTimeUnit": {
-              "type": "string",
-              "enum": [
-                "HOURS",
-                "DAYS",
-                "WEEKS"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "enum": [
+                    "HOURS",
+                    "DAYS",
+                    "WEEKS"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "jobs.PipelineParams": {
@@ -3795,15 +4035,23 @@
               ]
             },
             "jobs.RunIf": {
-              "type": "string",
-              "description": "An optional value indicating the condition that determines whether the task should be run once its dependencies have been completed. When omitted, defaults to `ALL_SUCCESS`.\n\nPossible values are:\n* `ALL_SUCCESS`: All dependencies have executed and succeeded\n* `AT_LEAST_ONE_SUCCESS`: At least one dependency has succeeded\n* `NONE_FAILED`: None of the dependencies have failed and at least one was executed\n* `ALL_DONE`: All dependencies have been completed\n* `AT_LEAST_ONE_FAILED`: At least one dependency failed\n* `ALL_FAILED`: ALl dependencies have failed",
-              "enum": [
-                "ALL_SUCCESS",
-                "ALL_DONE",
-                "NONE_FAILED",
-                "AT_LEAST_ONE_SUCCESS",
-                "ALL_FAILED",
-                "AT_LEAST_ONE_FAILED"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "An optional value indicating the condition that determines whether the task should be run once its dependencies have been completed. When omitted, defaults to `ALL_SUCCESS`.\n\nPossible values are:\n* `ALL_SUCCESS`: All dependencies have executed and succeeded\n* `AT_LEAST_ONE_SUCCESS`: At least one dependency has succeeded\n* `NONE_FAILED`: None of the dependencies have failed and at least one was executed\n* `ALL_DONE`: All dependencies have been completed\n* `AT_LEAST_ONE_FAILED`: At least one dependency failed\n* `ALL_FAILED`: ALl dependencies have failed",
+                  "enum": [
+                    "ALL_SUCCESS",
+                    "ALL_DONE",
+                    "NONE_FAILED",
+                    "AT_LEAST_ONE_SUCCESS",
+                    "ALL_FAILED",
+                    "AT_LEAST_ONE_FAILED"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "jobs.RunJobTask": {
@@ -3863,11 +4111,19 @@
               ]
             },
             "jobs.Source": {
-              "type": "string",
-              "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\\\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider.",
-              "enum": [
-                "WORKSPACE",
-                "GIT"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\\\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider.",
+                  "enum": [
+                    "WORKSPACE",
+                    "GIT"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "jobs.SparkJarTask": {
@@ -4567,12 +4823,20 @@
               ]
             },
             "ml.ModelVersionStatus": {
-              "type": "string",
-              "description": "Current status of `model_version`",
-              "enum": [
-                "PENDING_REGISTRATION",
-                "FAILED_REGISTRATION",
-                "READY"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Current status of `model_version`",
+                  "enum": [
+                    "PENDING_REGISTRATION",
+                    "FAILED_REGISTRATION",
+                    "READY"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "ml.ModelVersionTag": {
@@ -4618,23 +4882,39 @@
               ]
             },
             "pipelines.DayOfWeek": {
-              "type": "string",
-              "description": "Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour).\nIf not specified all days of the week will be used.",
-              "enum": [
-                "MONDAY",
-                "TUESDAY",
-                "WEDNESDAY",
-                "THURSDAY",
-                "FRIDAY",
-                "SATURDAY",
-                "SUNDAY"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour).\nIf not specified all days of the week will be used.",
+                  "enum": [
+                    "MONDAY",
+                    "TUESDAY",
+                    "WEDNESDAY",
+                    "THURSDAY",
+                    "FRIDAY",
+                    "SATURDAY",
+                    "SUNDAY"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "pipelines.DeploymentKind": {
-              "type": "string",
-              "description": "The deployment method that manages the pipeline:\n- BUNDLE: The pipeline is managed by a Databricks Asset Bundle.\n",
-              "enum": [
-                "BUNDLE"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "The deployment method that manages the pipeline:\n- BUNDLE: The pipeline is managed by a Databricks Asset Bundle.\n",
+                  "enum": [
+                    "BUNDLE"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "pipelines.FileLibrary": {
@@ -4940,11 +5220,19 @@
               ]
             },
             "pipelines.PipelineClusterAutoscaleMode": {
-              "type": "string",
-              "description": "Databricks Enhanced Autoscaling optimizes cluster utilization by automatically\nallocating cluster resources based on workload volume, with minimal impact to\nthe data processing latency of your pipelines. Enhanced Autoscaling is available\nfor `updates` clusters only. The legacy autoscaling feature is used for `maintenance`\nclusters.\n",
-              "enum": [
-                "ENHANCED",
-                "LEGACY"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Databricks Enhanced Autoscaling optimizes cluster utilization by automatically\nallocating cluster resources based on workload volume, with minimal impact to\nthe data processing latency of your pipelines. Enhanced Autoscaling is available\nfor `updates` clusters only. The legacy autoscaling feature is used for `maintenance`\nclusters.\n",
+                  "enum": [
+                    "ENHANCED",
+                    "LEGACY"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "pipelines.PipelineDeployment": {
@@ -5193,11 +5481,19 @@
               ]
             },
             "pipelines.TableSpecificConfigScdType": {
-              "type": "string",
-              "description": "The SCD type to use to ingest the table.",
-              "enum": [
-                "SCD_TYPE_1",
-                "SCD_TYPE_2"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "The SCD type to use to ingest the table.",
+                  "enum": [
+                    "SCD_TYPE_1",
+                    "SCD_TYPE_2"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "serving.Ai21LabsConfig": {
@@ -5304,11 +5600,19 @@
               ]
             },
             "serving.AiGatewayGuardrailPiiBehaviorBehavior": {
-              "type": "string",
-              "description": "Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.",
-              "enum": [
-                "NONE",
-                "BLOCK"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.",
+                  "enum": [
+                    "NONE",
+                    "BLOCK"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "serving.AiGatewayGuardrails": {
@@ -5394,18 +5698,34 @@
               ]
             },
             "serving.AiGatewayRateLimitKey": {
-              "type": "string",
-              "description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.",
-              "enum": [
-                "user",
-                "endpoint"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.",
+                  "enum": [
+                    "user",
+                    "endpoint"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "serving.AiGatewayRateLimitRenewalPeriod": {
-              "type": "string",
-              "description": "Renewal period field for a rate limit. Currently, only 'minute' is supported.",
-              "enum": [
-                "minute"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Renewal period field for a rate limit. Currently, only 'minute' is supported.",
+                  "enum": [
+                    "minute"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "serving.AiGatewayUsageTrackingConfig": {
@@ -5469,13 +5789,21 @@
               ]
             },
             "serving.AmazonBedrockConfigBedrockProvider": {
-              "type": "string",
-              "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.",
-              "enum": [
-                "anthropic",
-                "cohere",
-                "ai21labs",
-                "amazon"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.",
+                  "enum": [
+                    "anthropic",
+                    "cohere",
+                    "ai21labs",
+                    "amazon"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "serving.AnthropicConfig": {
@@ -5704,17 +6032,25 @@
               ]
             },
             "serving.ExternalModelProvider": {
-              "type": "string",
-              "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n",
-              "enum": [
-                "ai21labs",
-                "anthropic",
-                "amazon-bedrock",
-                "cohere",
-                "databricks-model-serving",
-                "google-cloud-vertex-ai",
-                "openai",
-                "palm"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n",
+                  "enum": [
+                    "ai21labs",
+                    "anthropic",
+                    "amazon-bedrock",
+                    "cohere",
+                    "databricks-model-serving",
+                    "google-cloud-vertex-ai",
+                    "openai",
+                    "palm"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "serving.GoogleCloudVertexAiConfig": {
@@ -5858,18 +6194,34 @@
               ]
             },
             "serving.RateLimitKey": {
-              "type": "string",
-              "description": "Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.",
-              "enum": [
-                "user",
-                "endpoint"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.",
+                  "enum": [
+                    "user",
+                    "endpoint"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "serving.RateLimitRenewalPeriod": {
-              "type": "string",
-              "description": "Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported.",
-              "enum": [
-                "minute"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported.",
+                  "enum": [
+                    "minute"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "serving.Route": {
@@ -6016,23 +6368,39 @@
               ]
             },
             "serving.ServedModelInputWorkloadSize": {
-              "type": "string",
-              "description": "The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.\n",
-              "enum": [
-                "Small",
-                "Medium",
-                "Large"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.\n",
+                  "enum": [
+                    "Small",
+                    "Medium",
+                    "Large"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "serving.ServedModelInputWorkloadType": {
-              "type": "string",
-              "description": "The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n",
-              "enum": [
-                "CPU",
-                "GPU_SMALL",
-                "GPU_MEDIUM",
-                "GPU_LARGE",
-                "MULTIGPU_MEDIUM"
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n",
+                  "enum": [
+                    "CPU",
+                    "GPU_SMALL",
+                    "GPU_MEDIUM",
+                    "GPU_LARGE",
+                    "MULTIGPU_MEDIUM"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
               ]
             },
             "serving.TrafficConfig": {

From e9902036b8149df9f72c6358a0450178b8ba965d Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 22 Jan 2025 11:50:13 +0100
Subject: [PATCH 124/247] Set WorktreeRoot to sync root outside git repo
 (#2197)

## Changes
If git is not detected, set default worktree root to sync root.
Otherwise NewFileSet/View raise an error about worktree root being
outside view root in acceptance/bundle/sync-paths-dotdot.

This behavior is introduced in
https://github.com/databricks/cli/pull/1945

Stacked on https://github.com/databricks/cli/pull/2202

## Tests
Existing tests.
---
 acceptance/bundle/syncroot/dotdot-nogit/output.txt | 6 +-----
 bundle/config/mutator/load_git_details.go          | 2 +-
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/acceptance/bundle/syncroot/dotdot-nogit/output.txt b/acceptance/bundle/syncroot/dotdot-nogit/output.txt
index 34059e276..46f617f35 100644
--- a/acceptance/bundle/syncroot/dotdot-nogit/output.txt
+++ b/acceptance/bundle/syncroot/dotdot-nogit/output.txt
@@ -1,11 +1,7 @@
-Error: path "$TMPDIR_PARENT" is not within repository root "$TMPDIR"
-
 Name: test-bundle
 Target: default
 Workspace:
   User: $USERNAME
   Path: /Workspace/Users/$USERNAME/.bundle/test-bundle/default
 
-Found 1 error
-
-Exit code: 1
+Validation OK!
diff --git a/bundle/config/mutator/load_git_details.go b/bundle/config/mutator/load_git_details.go
index 5c263ac03..3661c6bcd 100644
--- a/bundle/config/mutator/load_git_details.go
+++ b/bundle/config/mutator/load_git_details.go
@@ -32,7 +32,7 @@ func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn
 	}
 
 	if info.WorktreeRoot == "" {
-		b.WorktreeRoot = b.BundleRoot
+		b.WorktreeRoot = b.SyncRoot
 	} else {
 		b.WorktreeRoot = vfs.MustNew(info.WorktreeRoot)
 	}

From 876526a19a8f65e5ecc8fd22779dcefef8115c4a Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 22 Jan 2025 12:20:49 +0100
Subject: [PATCH 125/247] Use local git config in tests (#2205)

I've seen this error: could not lock config file
$TMPDIR_GPARENT/TestAccept3968313522/002/.gitconfig: File exists

This is likely the cause.
---
 acceptance/script.prepare | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acceptance/script.prepare b/acceptance/script.prepare
index 61061b59e..5900016d7 100644
--- a/acceptance/script.prepare
+++ b/acceptance/script.prepare
@@ -34,7 +34,7 @@ trace() {
 
 git-repo-init() {
     git init -qb main
-    git config --global core.autocrlf false
+    git config core.autocrlf false
     git config user.name "Tester"
     git config user.email "tester@databricks.com"
     git add databricks.yml

From 6c3ddbd921c568c3937d52a7167ae63fa3cbf533 Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Wed, 22 Jan 2025 17:44:54 +0530
Subject: [PATCH 126/247] Add `auth.Env` function (#2204)

## Changes
`auth.Env` is a generic function that we can use for authenticated tools
downstream to the CLI.

## Tests
Unit test.
---
 bundle/bundle.go      | 20 ++------------------
 libs/auth/env.go      | 26 ++++++++++++++++++++++++++
 libs/auth/env_test.go | 42 ++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 70 insertions(+), 18 deletions(-)
 create mode 100644 libs/auth/env.go
 create mode 100644 libs/auth/env_test.go

diff --git a/bundle/bundle.go b/bundle/bundle.go
index 3bf4ffb62..e715b8b2c 100644
--- a/bundle/bundle.go
+++ b/bundle/bundle.go
@@ -17,6 +17,7 @@ import (
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/env"
 	"github.com/databricks/cli/bundle/metadata"
+	"github.com/databricks/cli/libs/auth"
 	"github.com/databricks/cli/libs/fileset"
 	"github.com/databricks/cli/libs/locker"
 	"github.com/databricks/cli/libs/log"
@@ -24,7 +25,6 @@ import (
 	"github.com/databricks/cli/libs/terraform"
 	"github.com/databricks/cli/libs/vfs"
 	"github.com/databricks/databricks-sdk-go"
-	sdkconfig "github.com/databricks/databricks-sdk-go/config"
 	"github.com/hashicorp/terraform-exec/tfexec"
 )
 
@@ -242,21 +242,5 @@ func (b *Bundle) AuthEnv() (map[string]string, error) {
 	}
 
 	cfg := b.client.Config
-	out := make(map[string]string)
-	for _, attr := range sdkconfig.ConfigAttributes {
-		// Ignore profile so that downstream tools don't try and reload
-		// the profile even though we know the current configuration is valid.
-		if attr.Name == "profile" {
-			continue
-		}
-		if len(attr.EnvVars) == 0 {
-			continue
-		}
-		if attr.IsZero(cfg) {
-			continue
-		}
-		out[attr.EnvVars[0]] = attr.GetString(cfg)
-	}
-
-	return out, nil
+	return auth.Env(cfg), nil
 }
diff --git a/libs/auth/env.go b/libs/auth/env.go
new file mode 100644
index 000000000..c58cc53e3
--- /dev/null
+++ b/libs/auth/env.go
@@ -0,0 +1,26 @@
+package auth
+
+import "github.com/databricks/databricks-sdk-go/config"
+
+// Env generates the authentication environment variables we need to set for
+// downstream applications from the CLI to work correctly.
+func Env(cfg *config.Config) map[string]string {
+	out := make(map[string]string)
+	for _, attr := range config.ConfigAttributes {
+		// Ignore profile so that downstream tools don't try and reload
+		// the profile. We know the current configuration is already valid since
+		// otherwise the CLI would have thrown an error when loading it.
+		if attr.Name == "profile" {
+			continue
+		}
+		if len(attr.EnvVars) == 0 {
+			continue
+		}
+		if attr.IsZero(cfg) {
+			continue
+		}
+		out[attr.EnvVars[0]] = attr.GetString(cfg)
+	}
+
+	return out
+}
diff --git a/libs/auth/env_test.go b/libs/auth/env_test.go
new file mode 100644
index 000000000..be1cfc7ac
--- /dev/null
+++ b/libs/auth/env_test.go
@@ -0,0 +1,42 @@
+package auth
+
+import (
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/config"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAuthEnv(t *testing.T) {
+	in := &config.Config{
+		Profile:            "thisshouldbeignored",
+		Host:               "https://test.com",
+		Token:              "test-token",
+		Password:           "test-password",
+		MetadataServiceURL: "http://somurl.com",
+
+		AzureUseMSI:       true,
+		AzureTenantID:     "test-tenant-id",
+		AzureClientID:     "test-client-id",
+		AzureClientSecret: "test-client-secret",
+
+		ActionsIDTokenRequestToken: "test-actions-id-token-request-token",
+	}
+
+	expected := map[string]string{
+		"DATABRICKS_HOST":                 "https://test.com",
+		"DATABRICKS_TOKEN":                "test-token",
+		"DATABRICKS_PASSWORD":             "test-password",
+		"DATABRICKS_METADATA_SERVICE_URL": "http://somurl.com",
+
+		"ARM_USE_MSI":       "true",
+		"ARM_TENANT_ID":     "test-tenant-id",
+		"ARM_CLIENT_ID":     "test-client-id",
+		"ARM_CLIENT_SECRET": "test-client-secret",
+
+		"ACTIONS_ID_TOKEN_REQUEST_TOKEN": "test-actions-id-token-request-token",
+	}
+
+	out := Env(in)
+	assert.Equal(t, expected, out)
+}

From 667302b61b411fa56a6fa88395372fc227dae55f Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 22 Jan 2025 13:51:17 +0100
Subject: [PATCH 127/247] Refactor env forwarding function in terraform (#2206)

No functional changes, just making it easier to add variables.
---
 bundle/deploy/terraform/init.go | 36 ++++++++++++++-------------------
 1 file changed, 15 insertions(+), 21 deletions(-)

diff --git a/bundle/deploy/terraform/init.go b/bundle/deploy/terraform/init.go
index d982354e1..cc9cd4415 100644
--- a/bundle/deploy/terraform/init.go
+++ b/bundle/deploy/terraform/init.go
@@ -88,41 +88,35 @@ func (m *initialize) findExecPath(ctx context.Context, b *bundle.Bundle, tf *con
 	return tf.ExecPath, nil
 }
 
-// This function inherits some environment variables for Terraform CLI.
-func inheritEnvVars(ctx context.Context, environ map[string]string) error {
+var envCopy = []string{
 	// Include $HOME in set of environment variables to pass along.
-	home, ok := env.Lookup(ctx, "HOME")
-	if ok {
-		environ["HOME"] = home
-	}
+	"HOME",
 
 	// Include $USERPROFILE in set of environment variables to pass along.
 	// This variable is used by Azure CLI on Windows to find stored credentials and metadata
-	userProfile, ok := env.Lookup(ctx, "USERPROFILE")
-	if ok {
-		environ["USERPROFILE"] = userProfile
-	}
+	"USERPROFILE",
 
 	// Include $PATH in set of environment variables to pass along.
 	// This is necessary to ensure that our Terraform provider can use the
 	// same auxiliary programs (e.g. `az`, or `gcloud`) as the CLI.
-	path, ok := env.Lookup(ctx, "PATH")
-	if ok {
-		environ["PATH"] = path
-	}
+	"PATH",
 
 	// Include $AZURE_CONFIG_FILE in set of environment variables to pass along.
 	// This is set in Azure DevOps by the AzureCLI@2 task.
-	azureConfigFile, ok := env.Lookup(ctx, "AZURE_CONFIG_FILE")
-	if ok {
-		environ["AZURE_CONFIG_FILE"] = azureConfigFile
-	}
+	"AZURE_CONFIG_FILE",
 
 	// Include $TF_CLI_CONFIG_FILE to override terraform provider in development.
 	// See: https://developer.hashicorp.com/terraform/cli/config/config-file#explicit-installation-method-configuration
-	devConfigFile, ok := env.Lookup(ctx, "TF_CLI_CONFIG_FILE")
-	if ok {
-		environ["TF_CLI_CONFIG_FILE"] = devConfigFile
+	"TF_CLI_CONFIG_FILE",
+}
+
+// This function inherits some environment variables for Terraform CLI.
+func inheritEnvVars(ctx context.Context, environ map[string]string) error {
+	for _, key := range envCopy {
+		value, ok := env.Lookup(ctx, key)
+		if ok {
+			environ[key] = value
+		}
 	}
 
 	// Map $DATABRICKS_TF_CLI_CONFIG_FILE to $TF_CLI_CONFIG_FILE

From 54a470837ca0814955b4b1f8b41261645f4d546f Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 22 Jan 2025 14:28:13 +0100
Subject: [PATCH 128/247] Fix context propagation in bundle/deploy/terraform
 (#2208)

https://github.com/databricks/cli/pull/747#discussion_r1925248116
---
 bundle/deploy/terraform/init.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bundle/deploy/terraform/init.go b/bundle/deploy/terraform/init.go
index cc9cd4415..6a014a7c1 100644
--- a/bundle/deploy/terraform/init.go
+++ b/bundle/deploy/terraform/init.go
@@ -54,7 +54,7 @@ func (m *initialize) findExecPath(ctx context.Context, b *bundle.Bundle, tf *con
 		return tf.ExecPath, nil
 	}
 
-	binDir, err := b.CacheDir(context.Background(), "bin")
+	binDir, err := b.CacheDir(ctx, "bin")
 	if err != nil {
 		return "", err
 	}

From 3d91691f25e85220646df5b9bd8a672e28b14e7f Mon Sep 17 00:00:00 2001
From: Gleb Kanterov <kanterov@users.noreply.github.com>
Date: Wed, 22 Jan 2025 16:37:37 +0100
Subject: [PATCH 129/247] PythonMutator: propagate source locations (#1783)

## Changes
Add a mechanism to load Python source locations in the Python mutator.
Previously, locations pointed to generated YAML. Now, they point to
Python sources instead. Python process outputs "locations.json"
containing locations of bundle paths, examples:

```json
{"path": "resources.jobs.job_0", "file": "resources/job_0.py", "line": 3, "column": 5}
{"path": "resources.jobs.job_0.tasks[0].task_key", "file": "resources/job_0.py", "line": 10, "column": 5}
{"path": "resources.jobs.job_1", "file": "resources/job_1.py", "line": 5, "column": 7}
```

Such locations form a tree, and we assign locations of the closest
ancestor to each `dyn.Value` based on its path. For example,
`resources.jobs.job_0.tasks[0].task_key` is located at `job_0.py:10:5`
and `resources.jobs.job_0.tasks[0].email_notifications` is located at
`job_0.py:3:5`, because we use the location of the job as the most
precise approximation.

This feature is only enabled if `experimental/python` is used.

Note: for now, we don't update locations with relative paths, because it
has a side effect in changing how these paths are resolved

## Example
```
% databricks bundle validate

Warning: job_cluster_key abc is not defined
  at resources.jobs.examples.tasks[0].job_cluster_key
  in resources/example.py:10:1
```

## Tests
Unit tests and manually
---
 .../mutator/python/python_diagnostics.go      |   1 +
 .../config/mutator/python/python_locations.go | 194 ++++++++++++++++++
 .../mutator/python/python_locations_test.go   | 179 ++++++++++++++++
 .../config/mutator/python/python_mutator.go   | 101 +++++++--
 .../mutator/python/python_mutator_test.go     |  79 +++++--
 5 files changed, 518 insertions(+), 36 deletions(-)
 create mode 100644 bundle/config/mutator/python/python_locations.go
 create mode 100644 bundle/config/mutator/python/python_locations_test.go

diff --git a/bundle/config/mutator/python/python_diagnostics.go b/bundle/config/mutator/python/python_diagnostics.go
index 12822065b..7a1e13b4e 100644
--- a/bundle/config/mutator/python/python_diagnostics.go
+++ b/bundle/config/mutator/python/python_diagnostics.go
@@ -9,6 +9,7 @@ import (
 	"github.com/databricks/cli/libs/dyn"
 )
 
+// pythonDiagnostic is a single entry in diagnostics.json
 type pythonDiagnostic struct {
 	Severity pythonSeverity           `json:"severity"`
 	Summary  string                   `json:"summary"`
diff --git a/bundle/config/mutator/python/python_locations.go b/bundle/config/mutator/python/python_locations.go
new file mode 100644
index 000000000..2fa86bea0
--- /dev/null
+++ b/bundle/config/mutator/python/python_locations.go
@@ -0,0 +1,194 @@
+package python
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"path/filepath"
+
+	"github.com/databricks/cli/libs/dyn"
+)
+
+// generatedFileName is used as the virtual file name for YAML generated by Python code.
+//
+// mergePythonLocations replaces dyn.Location with generatedFileName with locations loaded
+// from locations.json
+const generatedFileName = "__generated_by_python__.yml"
+
+// pythonLocations is data structure for efficient location lookup for a given path
+//
+// Locations form a tree, and we assign locations of the closest ancestor to each dyn.Value based on its path.
+// We implement it as a trie (prefix tree) where keys are components of the path. With that, lookups are O(n)
+// where n is the number of components in the path.
+//
+// For example, with locations.json:
+//
+//		{"path": "resources.jobs.job_0", "file": "resources/job_0.py", "line": 3, "column": 5}
+//		{"path": "resources.jobs.job_0.tasks[0].task_key", "file": "resources/job_0.py", "line": 10, "column": 5}
+//		{"path": "resources.jobs.job_1", "file": "resources/job_1.py", "line": 5, "column": 7}
+//
+//	- resources.jobs.job_0.tasks[0].task_key is located at job_0.py:10:5
+//
+//	- resources.jobs.job_0.tasks[0].email_notifications is located at job_0.py:3:5,
+//	  because we use the location of the job as the most precise approximation.
+//
+// See pythonLocationEntry for the structure of a single entry in locations.json
+type pythonLocations struct {
+	// descendants referenced by index, e.g. '.foo'
+	keys map[string]*pythonLocations
+
+	// descendants referenced by key, e.g. '[0]'
+	indexes map[int]*pythonLocations
+
+	// location for the current node if it exists
+	location dyn.Location
+
+	// if true, location is present
+	exists bool
+}
+
+// pythonLocationEntry is a single entry in locations.json
+type pythonLocationEntry struct {
+	Path   string `json:"path"`
+	File   string `json:"file"`
+	Line   int    `json:"line"`
+	Column int    `json:"column"`
+}
+
+// mergePythonLocations applies locations from Python mutator into given dyn.Value
+//
+// The primary use-case is to merge locations.json with output.json, so that any
+// validation errors will point to Python source code instead of generated YAML.
+func mergePythonLocations(value dyn.Value, locations *pythonLocations) (dyn.Value, error) {
+	return dyn.Walk(value, func(path dyn.Path, value dyn.Value) (dyn.Value, error) {
+		newLocation, ok := findPythonLocation(locations, path)
+		if !ok {
+			return value, nil
+		}
+
+		// The first item in the list is the "last" location used for error reporting
+		//
+		// Loaded YAML uses virtual file path as location, we remove any of such references,
+		// because they should use 'newLocation' instead.
+		//
+		// We preserve any previous non-virtual locations in case when Python function modified
+		// resource defined in YAML.
+		newLocations := append(
+			[]dyn.Location{newLocation},
+			removeVirtualLocations(value.Locations())...,
+		)
+
+		return value.WithLocations(newLocations), nil
+	})
+}
+
+func removeVirtualLocations(locations []dyn.Location) []dyn.Location {
+	var newLocations []dyn.Location
+
+	for _, location := range locations {
+		if filepath.Base(location.File) == generatedFileName {
+			continue
+		}
+
+		newLocations = append(newLocations, location)
+	}
+
+	return newLocations
+}
+
+// parsePythonLocations parses locations.json from the Python mutator.
+//
+// locations file is newline-separated JSON objects with pythonLocationEntry structure.
+func parsePythonLocations(input io.Reader) (*pythonLocations, error) {
+	decoder := json.NewDecoder(input)
+	locations := newPythonLocations()
+
+	for decoder.More() {
+		var entry pythonLocationEntry
+
+		err := decoder.Decode(&entry)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse python location: %s", err)
+		}
+
+		path, err := dyn.NewPathFromString(entry.Path)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse python location: %s", err)
+		}
+
+		location := dyn.Location{
+			File:   entry.File,
+			Line:   entry.Line,
+			Column: entry.Column,
+		}
+
+		putPythonLocation(locations, path, location)
+	}
+
+	return locations, nil
+}
+
+// putPythonLocation puts the location to the trie for the given path
+func putPythonLocation(trie *pythonLocations, path dyn.Path, location dyn.Location) {
+	currentNode := trie
+
+	for _, component := range path {
+		if key := component.Key(); key != "" {
+			if _, ok := currentNode.keys[key]; !ok {
+				currentNode.keys[key] = newPythonLocations()
+			}
+
+			currentNode = currentNode.keys[key]
+		} else {
+			index := component.Index()
+			if _, ok := currentNode.indexes[index]; !ok {
+				currentNode.indexes[index] = newPythonLocations()
+			}
+
+			currentNode = currentNode.indexes[index]
+		}
+	}
+
+	currentNode.location = location
+	currentNode.exists = true
+}
+
+// newPythonLocations creates a new trie node
+func newPythonLocations() *pythonLocations {
+	return &pythonLocations{
+		keys:    make(map[string]*pythonLocations),
+		indexes: make(map[int]*pythonLocations),
+	}
+}
+
+// findPythonLocation finds the location or closest ancestor location in the trie for the given path
+// if no ancestor or exact location is found, false is returned.
+func findPythonLocation(locations *pythonLocations, path dyn.Path) (dyn.Location, bool) {
+	currentNode := locations
+	lastLocation := locations.location
+	exists := locations.exists
+
+	for _, component := range path {
+		if key := component.Key(); key != "" {
+			if _, ok := currentNode.keys[key]; !ok {
+				break
+			}
+
+			currentNode = currentNode.keys[key]
+		} else {
+			index := component.Index()
+			if _, ok := currentNode.indexes[index]; !ok {
+				break
+			}
+
+			currentNode = currentNode.indexes[index]
+		}
+
+		if currentNode.exists {
+			lastLocation = currentNode.location
+			exists = true
+		}
+	}
+
+	return lastLocation, exists
+}
diff --git a/bundle/config/mutator/python/python_locations_test.go b/bundle/config/mutator/python/python_locations_test.go
new file mode 100644
index 000000000..32afcc92b
--- /dev/null
+++ b/bundle/config/mutator/python/python_locations_test.go
@@ -0,0 +1,179 @@
+package python
+
+import (
+	"bytes"
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/cli/libs/diag"
+	"github.com/stretchr/testify/require"
+
+	"github.com/databricks/cli/libs/dyn"
+	assert "github.com/databricks/cli/libs/dyn/dynassert"
+)
+
+func TestMergeLocations(t *testing.T) {
+	pythonLocation := dyn.Location{File: "foo.py", Line: 1, Column: 1}
+	generatedLocation := dyn.Location{File: generatedFileName, Line: 1, Column: 1}
+	yamlLocation := dyn.Location{File: "foo.yml", Line: 1, Column: 1}
+
+	locations := newPythonLocations()
+	putPythonLocation(locations, dyn.MustPathFromString("foo"), pythonLocation)
+
+	input := dyn.NewValue(
+		map[string]dyn.Value{
+			"foo": dyn.NewValue(
+				map[string]dyn.Value{
+					"baz": dyn.NewValue("baz", []dyn.Location{yamlLocation}),
+					"qux": dyn.NewValue("baz", []dyn.Location{generatedLocation, yamlLocation}),
+				},
+				[]dyn.Location{},
+			),
+			"bar": dyn.NewValue("baz", []dyn.Location{generatedLocation}),
+		},
+		[]dyn.Location{yamlLocation},
+	)
+
+	expected := dyn.NewValue(
+		map[string]dyn.Value{
+			"foo": dyn.NewValue(
+				map[string]dyn.Value{
+					// pythonLocation is appended to the beginning of the list if absent
+					"baz": dyn.NewValue("baz", []dyn.Location{pythonLocation, yamlLocation}),
+					// generatedLocation is replaced by pythonLocation
+					"qux": dyn.NewValue("baz", []dyn.Location{pythonLocation, yamlLocation}),
+				},
+				[]dyn.Location{pythonLocation},
+			),
+			// if location is unknown, we keep it as-is
+			"bar": dyn.NewValue("baz", []dyn.Location{generatedLocation}),
+		},
+		[]dyn.Location{yamlLocation},
+	)
+
+	actual, err := mergePythonLocations(input, locations)
+
+	assert.NoError(t, err)
+	assert.Equal(t, expected, actual)
+}
+
+func TestFindLocation(t *testing.T) {
+	location0 := dyn.Location{File: "foo.py", Line: 1, Column: 1}
+	location1 := dyn.Location{File: "foo.py", Line: 2, Column: 1}
+
+	locations := newPythonLocations()
+	putPythonLocation(locations, dyn.MustPathFromString("foo"), location0)
+	putPythonLocation(locations, dyn.MustPathFromString("foo.bar"), location1)
+
+	actual, exists := findPythonLocation(locations, dyn.MustPathFromString("foo.bar"))
+
+	assert.True(t, exists)
+	assert.Equal(t, location1, actual)
+}
+
+func TestFindLocation_indexPathComponent(t *testing.T) {
+	location0 := dyn.Location{File: "foo.py", Line: 1, Column: 1}
+	location1 := dyn.Location{File: "foo.py", Line: 2, Column: 1}
+	location2 := dyn.Location{File: "foo.py", Line: 3, Column: 1}
+
+	locations := newPythonLocations()
+	putPythonLocation(locations, dyn.MustPathFromString("foo"), location0)
+	putPythonLocation(locations, dyn.MustPathFromString("foo.bar"), location1)
+	putPythonLocation(locations, dyn.MustPathFromString("foo.bar[0]"), location2)
+
+	actual, exists := findPythonLocation(locations, dyn.MustPathFromString("foo.bar[0]"))
+
+	assert.True(t, exists)
+	assert.Equal(t, location2, actual)
+}
+
+func TestFindLocation_closestAncestorLocation(t *testing.T) {
+	location0 := dyn.Location{File: "foo.py", Line: 1, Column: 1}
+	location1 := dyn.Location{File: "foo.py", Line: 2, Column: 1}
+
+	locations := newPythonLocations()
+	putPythonLocation(locations, dyn.MustPathFromString("foo"), location0)
+	putPythonLocation(locations, dyn.MustPathFromString("foo.bar"), location1)
+
+	actual, exists := findPythonLocation(locations, dyn.MustPathFromString("foo.bar.baz"))
+
+	assert.True(t, exists)
+	assert.Equal(t, location1, actual)
+}
+
+func TestFindLocation_unknownLocation(t *testing.T) {
+	location0 := dyn.Location{File: "foo.py", Line: 1, Column: 1}
+	location1 := dyn.Location{File: "foo.py", Line: 2, Column: 1}
+
+	locations := newPythonLocations()
+	putPythonLocation(locations, dyn.MustPathFromString("foo"), location0)
+	putPythonLocation(locations, dyn.MustPathFromString("foo.bar"), location1)
+
+	_, exists := findPythonLocation(locations, dyn.MustPathFromString("bar"))
+
+	assert.False(t, exists)
+}
+
+func TestLoadOutput(t *testing.T) {
+	location := dyn.Location{File: "my_job.py", Line: 1, Column: 1}
+	bundleRoot := t.TempDir()
+	output := `{
+		"resources": {
+			"jobs": {
+				"my_job": {
+					"name": "my_job",
+					"tasks": [
+						{
+							"task_key": "my_task",
+							"notebook_task": {
+								"notebook_path": "my_notebook"
+							}
+						}
+					]
+				}
+			}
+		}
+	}`
+
+	locations := newPythonLocations()
+	putPythonLocation(
+		locations,
+		dyn.MustPathFromString("resources.jobs.my_job"),
+		location,
+	)
+
+	value, diags := loadOutput(
+		bundleRoot,
+		bytes.NewReader([]byte(output)),
+		locations,
+	)
+
+	assert.Equal(t, diag.Diagnostics{}, diags)
+
+	name, err := dyn.Get(value, "resources.jobs.my_job.name")
+	require.NoError(t, err)
+	require.Equal(t, []dyn.Location{location}, name.Locations())
+
+	// until we implement path normalization, we have to keep locations of values
+	// that change semantic depending on their location
+	//
+	// note: it's important to have absolute path including 'bundleRoot'
+	// because mutator pipeline already has expanded locations into absolute path
+	notebookPath, err := dyn.Get(value, "resources.jobs.my_job.tasks[0].notebook_task.notebook_path")
+	require.NoError(t, err)
+	require.Len(t, notebookPath.Locations(), 1)
+	require.Equal(t, filepath.Join(bundleRoot, generatedFileName), notebookPath.Locations()[0].File)
+}
+
+func TestParsePythonLocations(t *testing.T) {
+	expected := dyn.Location{File: "foo.py", Line: 1, Column: 2}
+
+	input := `{"path": "foo", "file": "foo.py", "line": 1, "column": 2}`
+	reader := bytes.NewReader([]byte(input))
+	locations, err := parsePythonLocations(reader)
+
+	assert.NoError(t, err)
+
+	assert.True(t, locations.keys["foo"].exists)
+	assert.Equal(t, expected, locations.keys["foo"].location)
+}
diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go
index 8009ab243..cd2e286e5 100644
--- a/bundle/config/mutator/python/python_mutator.go
+++ b/bundle/config/mutator/python/python_mutator.go
@@ -7,11 +7,14 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"reflect"
 	"strings"
 
+	"github.com/databricks/cli/bundle/config/mutator/paths"
+
 	"github.com/databricks/databricks-sdk-go/logger"
 	"github.com/fatih/color"
 
@@ -124,6 +127,15 @@ type opts struct {
 	enabled bool
 
 	venvPath string
+
+	loadLocations bool
+}
+
+type runPythonMutatorOpts struct {
+	cacheDir       string
+	bundleRootPath string
+	pythonPath     string
+	loadLocations  bool
 }
 
 // getOpts adapts deprecated PyDABs and upcoming Python configuration
@@ -148,8 +160,9 @@ func getOpts(b *bundle.Bundle, phase phase) (opts, error) {
 		// don't execute for phases for 'python' section
 		if phase == PythonMutatorPhaseInit || phase == PythonMutatorPhaseLoad {
 			return opts{
-				enabled:  true,
-				venvPath: experimental.PyDABs.VEnvPath,
+				enabled:       true,
+				venvPath:      experimental.PyDABs.VEnvPath,
+				loadLocations: false, // not supported in PyDABs
 			}, nil
 		} else {
 			return opts{}, nil
@@ -158,8 +171,9 @@ func getOpts(b *bundle.Bundle, phase phase) (opts, error) {
 		// don't execute for phases for 'pydabs' section
 		if phase == PythonMutatorPhaseLoadResources || phase == PythonMutatorPhaseApplyMutators {
 			return opts{
-				enabled:  true,
-				venvPath: experimental.Python.VEnvPath,
+				enabled:       true,
+				venvPath:      experimental.Python.VEnvPath,
+				loadLocations: true,
 			}, nil
 		} else {
 			return opts{}, nil
@@ -194,7 +208,12 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
 			return dyn.InvalidValue, fmt.Errorf("failed to create cache dir: %w", err)
 		}
 
-		rightRoot, diags := m.runPythonMutator(ctx, cacheDir, b.BundleRootPath, pythonPath, leftRoot)
+		rightRoot, diags := m.runPythonMutator(ctx, leftRoot, runPythonMutatorOpts{
+			cacheDir:       cacheDir,
+			bundleRootPath: b.BundleRootPath,
+			pythonPath:     pythonPath,
+			loadLocations:  opts.loadLocations,
+		})
 		mutateDiags = diags
 		if diags.HasError() {
 			return dyn.InvalidValue, mutateDiagsHasError
@@ -238,13 +257,14 @@ func createCacheDir(ctx context.Context) (string, error) {
 	return os.MkdirTemp("", "-python")
 }
 
-func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir, rootPath, pythonPath string, root dyn.Value) (dyn.Value, diag.Diagnostics) {
-	inputPath := filepath.Join(cacheDir, "input.json")
-	outputPath := filepath.Join(cacheDir, "output.json")
-	diagnosticsPath := filepath.Join(cacheDir, "diagnostics.json")
+func (m *pythonMutator) runPythonMutator(ctx context.Context, root dyn.Value, opts runPythonMutatorOpts) (dyn.Value, diag.Diagnostics) {
+	inputPath := filepath.Join(opts.cacheDir, "input.json")
+	outputPath := filepath.Join(opts.cacheDir, "output.json")
+	diagnosticsPath := filepath.Join(opts.cacheDir, "diagnostics.json")
+	locationsPath := filepath.Join(opts.cacheDir, "locations.json")
 
 	args := []string{
-		pythonPath,
+		opts.pythonPath,
 		"-m",
 		"databricks.bundles.build",
 		"--phase",
@@ -257,6 +277,10 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir, rootPath
 		diagnosticsPath,
 	}
 
+	if opts.loadLocations {
+		args = append(args, "--locations", locationsPath)
+	}
+
 	if err := writeInputFile(inputPath, root); err != nil {
 		return dyn.InvalidValue, diag.Errorf("failed to write input file: %s", err)
 	}
@@ -271,7 +295,7 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir, rootPath
 	_, processErr := process.Background(
 		ctx,
 		args,
-		process.WithDir(rootPath),
+		process.WithDir(opts.bundleRootPath),
 		process.WithStderrWriter(stderrWriter),
 		process.WithStdoutWriter(stdoutWriter),
 	)
@@ -307,7 +331,12 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir, rootPath
 		return dyn.InvalidValue, diag.Errorf("failed to load diagnostics: %s", pythonDiagnosticsErr)
 	}
 
-	output, outputDiags := loadOutputFile(rootPath, outputPath)
+	locations, err := loadLocationsFile(locationsPath)
+	if err != nil {
+		return dyn.InvalidValue, diag.Errorf("failed to load locations: %s", err)
+	}
+
+	output, outputDiags := loadOutputFile(opts.bundleRootPath, outputPath, locations)
 	pythonDiagnostics = pythonDiagnostics.Extend(outputDiags)
 
 	// we pass through pythonDiagnostic because it contains warnings
@@ -351,7 +380,21 @@ func writeInputFile(inputPath string, input dyn.Value) error {
 	return os.WriteFile(inputPath, rootConfigJson, 0o600)
 }
 
-func loadOutputFile(rootPath, outputPath string) (dyn.Value, diag.Diagnostics) {
+// loadLocationsFile loads locations.json containing source locations for generated YAML.
+func loadLocationsFile(locationsPath string) (*pythonLocations, error) {
+	locationsFile, err := os.Open(locationsPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return newPythonLocations(), nil
+	} else if err != nil {
+		return nil, fmt.Errorf("failed to open locations file: %w", err)
+	}
+
+	defer locationsFile.Close()
+
+	return parsePythonLocations(locationsFile)
+}
+
+func loadOutputFile(rootPath, outputPath string, locations *pythonLocations) (dyn.Value, diag.Diagnostics) {
 	outputFile, err := os.Open(outputPath)
 	if err != nil {
 		return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to open output file: %w", err))
@@ -359,15 +402,19 @@ func loadOutputFile(rootPath, outputPath string) (dyn.Value, diag.Diagnostics) {
 
 	defer outputFile.Close()
 
+	return loadOutput(rootPath, outputFile, locations)
+}
+
+func loadOutput(rootPath string, outputFile io.Reader, locations *pythonLocations) (dyn.Value, diag.Diagnostics) {
 	// we need absolute path because later parts of pipeline assume all paths are absolute
 	// and this file will be used as location to resolve relative paths.
 	//
-	// virtualPath has to stay in rootPath, because locations outside root path are not allowed:
+	// virtualPath has to stay in bundleRootPath, because locations outside root path are not allowed:
 	//
 	//   Error: path /var/folders/.../python/dist/*.whl is not contained in bundle root path
 	//
 	// for that, we pass virtualPath instead of outputPath as file location
-	virtualPath, err := filepath.Abs(filepath.Join(rootPath, "__generated_by_python__.yml"))
+	virtualPath, err := filepath.Abs(filepath.Join(rootPath, generatedFileName))
 	if err != nil {
 		return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to get absolute path: %w", err))
 	}
@@ -377,7 +424,29 @@ func loadOutputFile(rootPath, outputPath string) (dyn.Value, diag.Diagnostics) {
 		return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to parse output file: %w", err))
 	}
 
-	return strictNormalize(config.Root{}, generated)
+	// paths are resolved relative to locations of their values, if we change location
+	// we have to update each path, until we simplify that, we don't update locations
+	// for such values, so we don't change how paths are resolved
+	//
+	// we can remove this once we:
+	// - add variable interpolation before and after PythonMutator
+	// - implement path normalization (aka path normal form)
+	_, err = paths.VisitJobPaths(generated, func(p dyn.Path, kind paths.PathKind, v dyn.Value) (dyn.Value, error) {
+		putPythonLocation(locations, p, v.Location())
+		return v, nil
+	})
+	if err != nil {
+		return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to update locations: %w", err))
+	}
+
+	// generated has dyn.Location as if it comes from generated YAML file
+	// earlier we loaded locations.json with source locations in Python code
+	generatedWithLocations, err := mergePythonLocations(generated, locations)
+	if err != nil {
+		return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to update locations: %w", err))
+	}
+
+	return strictNormalize(config.Root{}, generatedWithLocations)
 }
 
 func strictNormalize(dst any, generated dyn.Value) (dyn.Value, diag.Diagnostics) {
diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go
index d51572c8a..322fb79e8 100644
--- a/bundle/config/mutator/python/python_mutator_test.go
+++ b/bundle/config/mutator/python/python_mutator_test.go
@@ -7,7 +7,6 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"reflect"
 	"runtime"
 	"testing"
 
@@ -93,6 +92,8 @@ func TestPythonMutator_loadResources(t *testing.T) {
 			}
 		}`,
 		`{"severity": "warning", "summary": "job doesn't have any tasks", "location": {"file": "src/examples/file.py", "line": 10, "column": 5}}`,
+		`{"path": "resources.jobs.job0", "file": "src/examples/job0.py", "line": 3, "column": 5}
+		{"path": "resources.jobs.job1", "file": "src/examples/job1.py", "line": 5, "column": 7}`,
 	)
 
 	mutator := PythonMutator(PythonMutatorPhaseLoadResources)
@@ -110,6 +111,25 @@ func TestPythonMutator_loadResources(t *testing.T) {
 		assert.Equal(t, "job_1", job1.Name)
 	}
 
+	// output of locations.json should be applied to underlying dyn.Value
+	err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
+		name1, err := dyn.GetByPath(v, dyn.MustPathFromString("resources.jobs.job1.name"))
+		if err != nil {
+			return dyn.InvalidValue, err
+		}
+
+		assert.Equal(t, []dyn.Location{
+			{
+				File:   "src/examples/job1.py",
+				Line:   5,
+				Column: 7,
+			},
+		}, name1.Locations())
+
+		return v, nil
+	})
+	assert.NoError(t, err)
+
 	assert.Equal(t, 1, len(diags))
 	assert.Equal(t, "job doesn't have any tasks", diags[0].Summary)
 	assert.Equal(t, []dyn.Location{
@@ -157,7 +177,7 @@ func TestPythonMutator_loadResources_disallowed(t *testing.T) {
 					}
 				}
 			}
-		}`, "")
+		}`, "", "")
 
 	mutator := PythonMutator(PythonMutatorPhaseLoadResources)
 	diag := bundle.Apply(ctx, b, mutator)
@@ -202,7 +222,7 @@ func TestPythonMutator_applyMutators(t *testing.T) {
 					}
 				}
 			}
-		}`, "")
+		}`, "", "")
 
 	mutator := PythonMutator(PythonMutatorPhaseApplyMutators)
 	diag := bundle.Apply(ctx, b, mutator)
@@ -224,7 +244,7 @@ func TestPythonMutator_applyMutators(t *testing.T) {
 		description, err := dyn.GetByPath(v, dyn.MustPathFromString("resources.jobs.job0.description"))
 		require.NoError(t, err)
 
-		expectedVirtualPath, err := filepath.Abs("__generated_by_python__.yml")
+		expectedVirtualPath, err := filepath.Abs(generatedFileName)
 		require.NoError(t, err)
 		assert.Equal(t, expectedVirtualPath, description.Location().File)
 
@@ -263,7 +283,7 @@ func TestPythonMutator_badOutput(t *testing.T) {
 					}
 				}
 			}
-		}`, "")
+		}`, "", "")
 
 	mutator := PythonMutator(PythonMutatorPhaseLoadResources)
 	diag := bundle.Apply(ctx, b, mutator)
@@ -312,7 +332,7 @@ func TestGetOps_Python(t *testing.T) {
 	}, PythonMutatorPhaseLoadResources)
 
 	assert.NoError(t, err)
-	assert.Equal(t, opts{venvPath: ".venv", enabled: true}, actual)
+	assert.Equal(t, opts{venvPath: ".venv", enabled: true, loadLocations: true}, actual)
 }
 
 func TestGetOps_PyDABs(t *testing.T) {
@@ -328,7 +348,7 @@ func TestGetOps_PyDABs(t *testing.T) {
 	}, PythonMutatorPhaseInit)
 
 	assert.NoError(t, err)
-	assert.Equal(t, opts{venvPath: ".venv", enabled: true}, actual)
+	assert.Equal(t, opts{venvPath: ".venv", enabled: true, loadLocations: false}, actual)
 }
 
 func TestGetOps_empty(t *testing.T) {
@@ -661,7 +681,7 @@ or activate the environment before running CLI commands:
 	assert.Equal(t, expected, out)
 }
 
-func withProcessStub(t *testing.T, args []string, output, diagnostics string) context.Context {
+func withProcessStub(t *testing.T, args []string, output, diagnostics, locations string) context.Context {
 	ctx := context.Background()
 	ctx, stub := process.WithStub(ctx)
 
@@ -673,32 +693,51 @@ func withProcessStub(t *testing.T, args []string, output, diagnostics string) co
 
 	inputPath := filepath.Join(cacheDir, "input.json")
 	outputPath := filepath.Join(cacheDir, "output.json")
+	locationsPath := filepath.Join(cacheDir, "locations.json")
 	diagnosticsPath := filepath.Join(cacheDir, "diagnostics.json")
 
-	args = append(args, "--input", inputPath)
-	args = append(args, "--output", outputPath)
-	args = append(args, "--diagnostics", diagnosticsPath)
-
 	stub.WithCallback(func(actual *exec.Cmd) error {
 		_, err := os.Stat(inputPath)
 		assert.NoError(t, err)
 
-		if reflect.DeepEqual(actual.Args, args) {
-			err := os.WriteFile(outputPath, []byte(output), 0o600)
-			require.NoError(t, err)
+		actualInputPath := getArg(actual.Args, "--input")
+		actualOutputPath := getArg(actual.Args, "--output")
+		actualDiagnosticsPath := getArg(actual.Args, "--diagnostics")
+		actualLocationsPath := getArg(actual.Args, "--locations")
 
-			err = os.WriteFile(diagnosticsPath, []byte(diagnostics), 0o600)
-			require.NoError(t, err)
+		require.Equal(t, inputPath, actualInputPath)
+		require.Equal(t, outputPath, actualOutputPath)
+		require.Equal(t, diagnosticsPath, actualDiagnosticsPath)
 
-			return nil
-		} else {
-			return fmt.Errorf("unexpected command: %v", actual.Args)
+		// locations is an optional argument
+		if locations != "" {
+			require.Equal(t, locationsPath, actualLocationsPath)
+
+			err = os.WriteFile(locationsPath, []byte(locations), 0o600)
+			require.NoError(t, err)
 		}
+
+		err = os.WriteFile(outputPath, []byte(output), 0o600)
+		require.NoError(t, err)
+
+		err = os.WriteFile(diagnosticsPath, []byte(diagnostics), 0o600)
+		require.NoError(t, err)
+
+		return nil
 	})
 
 	return ctx
 }
 
+func getArg(args []string, name string) string {
+	for i := range args {
+		if args[i] == name {
+			return args[i+1]
+		}
+	}
+	return ""
+}
+
 func loadYaml(name, content string) *bundle.Bundle {
 	v, diag := config.LoadFromBytes(name, []byte(content))
 

From 20c1902a4515ea464d03f6b9a8c2e7ede94e0ab4 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 22 Jan 2025 17:26:16 +0100
Subject: [PATCH 130/247] Fix passing SingleTest to TestAccept (#2210)

---
 acceptance/acceptance_test.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 9a4564ffa..e611f4e50 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -28,8 +28,8 @@ var KeepTmp bool
 
 // In order to debug CLI running under acceptance test, set this to full subtest name, e.g. "bundle/variables/empty"
 // Then install your breakpoints and click "debug test" near TestAccept in VSCODE.
-// example: var singleTest = "bundle/variables/empty"
-var singleTest = ""
+// example: var SingleTest = "bundle/variables/empty"
+var SingleTest = ""
 
 // If enabled, instead of compiling and running CLI externally, we'll start in-process server that accepts and runs
 // CLI commands. The $CLI in test scripts is a helper that just forwards command-line arguments to this server (see bin/callserver.py).
@@ -37,7 +37,7 @@ var singleTest = ""
 var InprocessMode bool
 
 func init() {
-	flag.BoolVar(&InprocessMode, "inprocess", singleTest != "", "Run CLI in the same process as test (for debugging)")
+	flag.BoolVar(&InprocessMode, "inprocess", SingleTest != "", "Run CLI in the same process as test (for debugging)")
 	flag.BoolVar(&KeepTmp, "keeptmp", false, "Do not delete TMP directory after run")
 }
 
@@ -54,7 +54,7 @@ var Scripts = map[string]bool{
 }
 
 func TestAccept(t *testing.T) {
-	testAccept(t, InprocessMode, "")
+	testAccept(t, InprocessMode, SingleTest)
 }
 
 func TestInprocessMode(t *testing.T) {

From ba3a400327833caa822c8a0416808b072c86c264 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 23 Jan 2025 11:59:01 +0100
Subject: [PATCH 131/247] Remove test-specific logic from generic test runner
 (#2215)

Revert changes to acceptance_test.go added in #2177 and add
test-specific fix.
---
 acceptance/acceptance_test.go                 | 23 -------------------
 .../experimental-jobs-as-code/script          |  2 ++
 2 files changed, 2 insertions(+), 23 deletions(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index e611f4e50..56db6ec20 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -9,7 +9,6 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"regexp"
 	"runtime"
 	"slices"
 	"sort"
@@ -452,16 +451,6 @@ func CopyDir(src, dst string, inputs, outputs map[string]bool) error {
 }
 
 func ListDir(t *testing.T, src string) ([]string, error) {
-	// exclude folders in .gitignore from comparison
-	ignored := []string{
-		"\\.ruff_cache",
-		"\\.venv",
-		".*\\.egg-info",
-		"__pycache__",
-		// depends on uv version
-		"uv.lock",
-	}
-
 	var files []string
 	err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -469,19 +458,7 @@ func ListDir(t *testing.T, src string) ([]string, error) {
 		}
 
 		if info.IsDir() {
-			for _, ignoredFolder := range ignored {
-				if matched, _ := regexp.MatchString(ignoredFolder, info.Name()); matched {
-					return filepath.SkipDir
-				}
-			}
-
 			return nil
-		} else {
-			for _, ignoredFolder := range ignored {
-				if matched, _ := regexp.MatchString(ignoredFolder, info.Name()); matched {
-					return nil
-				}
-			}
 		}
 
 		relPath, err := filepath.Rel(src, path)
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/script b/acceptance/bundle/templates/experimental-jobs-as-code/script
index 2209aa7ab..af28b9d0a 100644
--- a/acceptance/bundle/templates/experimental-jobs-as-code/script
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/script
@@ -10,3 +10,5 @@ cat databricks.yml | grep -v databricks_cli_version > databricks.yml.new
 mv databricks.yml.new databricks.yml
 
 trace $CLI bundle validate -t dev --output json | jq ".resources"
+
+rm -fr .venv resources/__pycache__ uv.lock my_jobs_as_code.egg-info

From f60ad32f07241b311192c1476e32ef8656e3c6f2 Mon Sep 17 00:00:00 2001
From: Ilya Kuznetsov <ilya.kuznetsov@databricks.com>
Date: Thu, 23 Jan 2025 12:11:44 +0100
Subject: [PATCH 132/247] Allow yaml-anchors in schema (#2200)

## Changes

Allows custom untyped fields in the root config in json-schema so it
doesn't highlight errors when using yaml-anchors.

Example use case:

```
tags: &job-tags
  environment: ${bundle.target}


resources:
  jobs:
    db1:
      tags:
        <<: *job-tags
    db1:
      tags:
        <<: *job-tags
```

One downside is that we don't highlight any unknown top-level properties
anymore (but they will still fail during CLI validation)

## Tests

Manually checked behavior in VSCode - it doesn't show validation error.
Also checked that other typed properties are still suggested
---
 bundle/internal/schema/main.go                        |  9 +++++++++
 .../schema/testdata/fail/unknown_top_level_field.yml  |  1 -
 bundle/internal/schema/testdata/pass/yaml_anchors.yml | 11 +++++++++++
 bundle/schema/jsonschema.json                         |  2 +-
 4 files changed, 21 insertions(+), 2 deletions(-)
 delete mode 100644 bundle/internal/schema/testdata/fail/unknown_top_level_field.yml
 create mode 100644 bundle/internal/schema/testdata/pass/yaml_anchors.yml

diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go
index 39b859656..38e099ece 100644
--- a/bundle/internal/schema/main.go
+++ b/bundle/internal/schema/main.go
@@ -172,6 +172,15 @@ func generateSchema(workdir, outputFile string) {
 		a.addAnnotations,
 		addInterpolationPatterns,
 	})
+
+	// AdditionalProperties is set to an empty schema to allow non-typed keys used as yaml-anchors
+	// Example:
+	// some_anchor: &some_anchor
+	//   file_path: /some/path/
+	// workspace:
+	//   <<: *some_anchor
+	s.AdditionalProperties = jsonschema.Schema{}
+
 	if err != nil {
 		log.Fatal(err)
 	}
diff --git a/bundle/internal/schema/testdata/fail/unknown_top_level_field.yml b/bundle/internal/schema/testdata/fail/unknown_top_level_field.yml
deleted file mode 100644
index e8a8866bc..000000000
--- a/bundle/internal/schema/testdata/fail/unknown_top_level_field.yml
+++ /dev/null
@@ -1 +0,0 @@
-unknown: value
diff --git a/bundle/internal/schema/testdata/pass/yaml_anchors.yml b/bundle/internal/schema/testdata/pass/yaml_anchors.yml
new file mode 100644
index 000000000..18749891d
--- /dev/null
+++ b/bundle/internal/schema/testdata/pass/yaml_anchors.yml
@@ -0,0 +1,11 @@
+tags: &job-tags
+  environment: "some_environment"
+
+resources:
+  jobs:
+    db1:
+      tags:
+        <<: *job-tags
+    db2:
+      tags:
+        <<: *job-tags
diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json
index b3158792c..4a3b56814 100644
--- a/bundle/schema/jsonschema.json
+++ b/bundle/schema/jsonschema.json
@@ -7269,5 +7269,5 @@
       "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace"
     }
   },
-  "additionalProperties": false
+  "additionalProperties": {}
 }
\ No newline at end of file

From 798189eb96bc1184119dc039a2728f87b4ce6212 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Thu, 23 Jan 2025 12:17:52 +0100
Subject: [PATCH 133/247] Upgrade Go SDK to 0.56.0 (#2214)

## Changes

Upgrade Go SDK to 0.56.0

Relevant changes:
- Support Query parameters for all HTTP operations
(https://github.com/databricks/databricks-sdk-go/pull/1124).
---
 .codegen/_openapi_sha                         |   2 +-
 .codegen/service.go.tmpl                      |  20 +-
 .gitattributes                                |   1 +
 bundle/deploy/terraform/convert_test.go       |   4 +-
 .../convert_model_serving_endpoint_test.go    |   2 +-
 .../internal/schema/annotations_openapi.yml   | 367 +++++++++++-------
 .../schema/annotations_openapi_overrides.yml  |  11 +
 bundle/schema/jsonschema.json                 | 175 +++++----
 .../custom-app-integration.go                 |   1 +
 cmd/api/api.go                                |   2 +-
 .../access-control/access-control.go          | 109 ++++++
 cmd/workspace/cmd.go                          |   2 +
 cmd/workspace/providers/providers.go          |   4 +-
 cmd/workspace/recipients/recipients.go        |  96 ++---
 .../serving-endpoints/serving-endpoints.go    | 111 +++++-
 go.mod                                        |   2 +-
 go.sum                                        |   4 +-
 integration/cmd/sync/sync_test.go             |   2 +-
 libs/filer/files_client.go                    |   4 +-
 libs/filer/workspace_files_client.go          |   5 +-
 .../workspace_files_extensions_client_test.go |   2 +-
 libs/git/info.go                              |   1 +
 22 files changed, 588 insertions(+), 339 deletions(-)
 create mode 100755 cmd/workspace/access-control/access-control.go

diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha
index dfe78790a..588cf9d63 100644
--- a/.codegen/_openapi_sha
+++ b/.codegen/_openapi_sha
@@ -1 +1 @@
-779817ed8d63031f5ea761fbd25ee84f38feec0d
\ No newline at end of file
+0be1b914249781b5e903b7676fd02255755bc851
\ No newline at end of file
diff --git a/.codegen/service.go.tmpl b/.codegen/service.go.tmpl
index 0c9fa089a..2f4987b13 100644
--- a/.codegen/service.go.tmpl
+++ b/.codegen/service.go.tmpl
@@ -109,16 +109,19 @@ var {{.CamelName}}Overrides []func(
 	{{- end }}
 )
 
+{{- $excludeFromJson := list "http-request"}}
+
 func new{{.PascalName}}() *cobra.Command {
 	cmd := &cobra.Command{}
 
+	{{- $canUseJson := and .CanUseJson (not (in $excludeFromJson .KebabName )) -}}
 	{{- if .Request}}
 
 	var {{.CamelName}}Req {{.Service.Package.Name}}.{{.Request.PascalName}}
 	{{- if .RequestBodyField }}
 	{{.CamelName}}Req.{{.RequestBodyField.PascalName}} = &{{.Service.Package.Name}}.{{.RequestBodyField.Entity.PascalName}}{}
 	{{- end }}
-	{{- if .CanUseJson}}
+	{{- if $canUseJson}}
 	var {{.CamelName}}Json flags.JsonFlag
 	{{- end}}
 	{{- end}}
@@ -135,7 +138,7 @@ func new{{.PascalName}}() *cobra.Command {
 	{{- $request = .RequestBodyField.Entity -}}
 	{{- end -}}
     {{if $request }}// TODO: short flags
-	{{- if .CanUseJson}}
+	{{- if $canUseJson}}
 	cmd.Flags().Var(&{{.CamelName}}Json, "json", `either inline JSON string or @path/to/file.json with request body`)
 	{{- end}}
     {{$method := .}}
@@ -177,7 +180,7 @@ func new{{.PascalName}}() *cobra.Command {
 	{{- $hasRequiredArgs :=  and (not $hasIdPrompt) $hasPosArgs -}}
 	{{- $hasSingleRequiredRequestBodyFieldWithPrompt := and (and $hasIdPrompt $request) (eq 1 (len $request.RequiredRequestBodyFields))  -}}
 	{{- $onlyPathArgsRequiredAsPositionalArguments := and $request (eq (len .RequiredPositionalArguments) (len $request.RequiredPathFields)) -}}
-	{{- $hasDifferentArgsWithJsonFlag := and (not $onlyPathArgsRequiredAsPositionalArguments) (and .CanUseJson (or $request.HasRequiredRequestBodyFields )) -}}
+	{{- $hasDifferentArgsWithJsonFlag := and (not $onlyPathArgsRequiredAsPositionalArguments) (and $canUseJson (or $request.HasRequiredRequestBodyFields )) -}}
 	{{- $hasCustomArgHandler := or $hasRequiredArgs $hasDifferentArgsWithJsonFlag -}}
 
 	{{- $atleastOneArgumentWithDescription := false -}}
@@ -239,7 +242,7 @@ func new{{.PascalName}}() *cobra.Command {
 		ctx := cmd.Context()
 		{{if .Service.IsAccounts}}a := root.AccountClient(ctx){{else}}w := root.WorkspaceClient(ctx){{end}}
 		{{- if .Request }}
-			{{ if .CanUseJson }}
+			{{ if $canUseJson }}
 			if cmd.Flags().Changed("json") {
 					diags := {{.CamelName}}Json.Unmarshal(&{{.CamelName}}Req{{ if .RequestBodyField }}.{{.RequestBodyField.PascalName}}{{ end }})
 					if diags.HasError() {
@@ -255,7 +258,7 @@ func new{{.PascalName}}() *cobra.Command {
 				return fmt.Errorf("please provide command input in JSON format by specifying the --json flag")
 			}{{- end}}
 			{{- if $hasPosArgs }}
-			{{- if and .CanUseJson $hasSingleRequiredRequestBodyFieldWithPrompt }} else {
+			{{- if and $canUseJson $hasSingleRequiredRequestBodyFieldWithPrompt }} else {
 			{{- end}}
 			{{- if $hasIdPrompt}}
 				if len(args) == 0 {
@@ -279,9 +282,9 @@ func new{{.PascalName}}() *cobra.Command {
 
 			{{$method := .}}
 			{{- range $arg, $field := .RequiredPositionalArguments}}
-				{{- template "args-scan" (dict "Arg" $arg "Field" $field "Method" $method "HasIdPrompt" $hasIdPrompt)}}
+				{{- template "args-scan" (dict "Arg" $arg "Field" $field "Method" $method "HasIdPrompt" $hasIdPrompt "ExcludeFromJson" $excludeFromJson)}}
 			{{- end -}}
-			{{- if and .CanUseJson $hasSingleRequiredRequestBodyFieldWithPrompt }}
+			{{- if and $canUseJson $hasSingleRequiredRequestBodyFieldWithPrompt }}
 			}
 			{{- end}}
 
@@ -392,7 +395,8 @@ func new{{.PascalName}}() *cobra.Command {
 	{{- $method := .Method -}}
 	{{- $arg := .Arg -}}
 	{{- $hasIdPrompt := .HasIdPrompt -}}
-	{{- $optionalIfJsonIsUsed := and (not $hasIdPrompt) (and $field.IsRequestBodyField $method.CanUseJson) }}
+	{{ $canUseJson := and $method.CanUseJson (not (in .ExcludeFromJson $method.KebabName)) }}
+	{{- $optionalIfJsonIsUsed := and (not $hasIdPrompt) (and $field.IsRequestBodyField $canUseJson) }}
 	{{- if $optionalIfJsonIsUsed  }}
 	if !cmd.Flags().Changed("json") {
 	{{- end }}
diff --git a/.gitattributes b/.gitattributes
index 0a8ddf3cb..ebe94ed8e 100755
--- a/.gitattributes
+++ b/.gitattributes
@@ -31,6 +31,7 @@ cmd/account/users/users.go linguist-generated=true
 cmd/account/vpc-endpoints/vpc-endpoints.go linguist-generated=true
 cmd/account/workspace-assignment/workspace-assignment.go linguist-generated=true
 cmd/account/workspaces/workspaces.go linguist-generated=true
+cmd/workspace/access-control/access-control.go linguist-generated=true
 cmd/workspace/aibi-dashboard-embedding-access-policy/aibi-dashboard-embedding-access-policy.go linguist-generated=true
 cmd/workspace/aibi-dashboard-embedding-approved-domains/aibi-dashboard-embedding-approved-domains.go linguist-generated=true
 cmd/workspace/alerts-legacy/alerts-legacy.go linguist-generated=true
diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go
index ffe55db71..afc1fb22a 100644
--- a/bundle/deploy/terraform/convert_test.go
+++ b/bundle/deploy/terraform/convert_test.go
@@ -419,7 +419,7 @@ func TestBundleToTerraformModelServing(t *testing.T) {
 	src := resources.ModelServingEndpoint{
 		CreateServingEndpoint: &serving.CreateServingEndpoint{
 			Name: "name",
-			Config: serving.EndpointCoreConfigInput{
+			Config: &serving.EndpointCoreConfigInput{
 				ServedModels: []serving.ServedModelInput{
 					{
 						ModelName:          "model_name",
@@ -474,7 +474,7 @@ func TestBundleToTerraformModelServingPermissions(t *testing.T) {
 			// and as such observed the `omitempty` tag.
 			// The new method leverages [dyn.Value] where any field that is not
 			// explicitly set is not part of the value.
-			Config: serving.EndpointCoreConfigInput{
+			Config: &serving.EndpointCoreConfigInput{
 				ServedModels: []serving.ServedModelInput{
 					{
 						ModelName:          "model_name",
diff --git a/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go b/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go
index d46350bb7..98cf2dc22 100644
--- a/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go
+++ b/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go
@@ -17,7 +17,7 @@ func TestConvertModelServingEndpoint(t *testing.T) {
 	src := resources.ModelServingEndpoint{
 		CreateServingEndpoint: &serving.CreateServingEndpoint{
 			Name: "name",
-			Config: serving.EndpointCoreConfigInput{
+			Config: &serving.EndpointCoreConfigInput{
 				ServedModels: []serving.ServedModelInput{
 					{
 						ModelName:          "model_name",
diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml
index 8ff5c9253..d5a9bf69e 100644
--- a/bundle/internal/schema/annotations_openapi.yml
+++ b/bundle/internal/schema/annotations_openapi.yml
@@ -353,12 +353,12 @@ github.com/databricks/cli/bundle/config/resources.MlflowModel:
 github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint:
   "ai_gateway":
     "description": |-
-      The AI Gateway configuration for the serving endpoint. NOTE: only external model endpoints are supported as of now.
+      The AI Gateway configuration for the serving endpoint. NOTE: Only external model and provisioned throughput endpoints are currently supported.
   "config":
     "description": |-
       The core config of the serving endpoint.
   "name":
-    "description": |
+    "description": |-
       The name of the serving endpoint. This field is required and must be unique across a Databricks workspace.
       An endpoint name can consist of alphanumeric characters, dashes, and underscores.
   "rate_limits":
@@ -1974,6 +1974,9 @@ github.com/databricks/databricks-sdk-go/service/jobs.SparkJarTask:
       Parameters passed to the main method.
 
       Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.
+  "run_as_repl":
+    "description": |-
+      Deprecated. A value of `false` is no longer supported.
 github.com/databricks/databricks-sdk-go/service/jobs.SparkPythonTask:
   "parameters":
     "description": |-
@@ -2684,27 +2687,36 @@ github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfigScd
 github.com/databricks/databricks-sdk-go/service/serving.Ai21LabsConfig:
   "ai21labs_api_key":
     "description": |-
-      The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`.
+      The Databricks secret key reference for an AI21 Labs API key. If you
+      prefer to paste your API key directly, see `ai21labs_api_key_plaintext`.
+      You must provide an API key using one of the following fields:
+      `ai21labs_api_key` or `ai21labs_api_key_plaintext`.
   "ai21labs_api_key_plaintext":
     "description": |-
-      An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`.
+      An AI21 Labs API key provided as a plaintext string. If you prefer to
+      reference your key using Databricks Secrets, see `ai21labs_api_key`. You
+      must provide an API key using one of the following fields:
+      `ai21labs_api_key` or `ai21labs_api_key_plaintext`.
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayConfig:
   "guardrails":
     "description": |-
       Configuration for AI Guardrails to prevent unwanted data and unsafe data in requests and responses.
   "inference_table_config":
     "description": |-
-      Configuration for payload logging using inference tables. Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.
+      Configuration for payload logging using inference tables.
+      Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.
   "rate_limits":
     "description": |-
       Configuration for rate limits which can be set to limit endpoint traffic.
   "usage_tracking_config":
     "description": |-
-      Configuration to enable usage tracking using system tables. These tables allow you to monitor operational usage on endpoints and their associated costs.
+      Configuration to enable usage tracking using system tables.
+      These tables allow you to monitor operational usage on endpoints and their associated costs.
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParameters:
   "invalid_keywords":
     "description": |-
-      List of invalid keywords. AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.
+      List of invalid keywords.
+      AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.
   "pii":
     "description": |-
       Configuration for guardrail PII filter.
@@ -2713,15 +2725,14 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParame
       Indicates whether the safety filter is enabled.
   "valid_topics":
     "description": |-
-      The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics.
+      The list of allowed topics.
+      Given a chat request, this guardrail flags the request if its topic is not in the allowed topics.
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehavior:
   "behavior":
     "description": |-
-      Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.
+      Configuration for input guardrail filters.
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehaviorBehavior:
   "_":
-    "description": |-
-      Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.
     "enum":
       - |-
         NONE
@@ -2737,30 +2748,32 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrails:
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayInferenceTableConfig:
   "catalog_name":
     "description": |-
-      The name of the catalog in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the catalog name.
+      The name of the catalog in Unity Catalog. Required when enabling inference tables.
+      NOTE: On update, you have to disable inference table first in order to change the catalog name.
   "enabled":
     "description": |-
       Indicates whether the inference table is enabled.
   "schema_name":
     "description": |-
-      The name of the schema in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the schema name.
+      The name of the schema in Unity Catalog. Required when enabling inference tables.
+      NOTE: On update, you have to disable inference table first in order to change the schema name.
   "table_name_prefix":
     "description": |-
-      The prefix of the table in Unity Catalog. NOTE: On update, you have to disable inference table first in order to change the prefix name.
+      The prefix of the table in Unity Catalog.
+      NOTE: On update, you have to disable inference table first in order to change the prefix name.
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit:
   "calls":
     "description": |-
       Used to specify how many calls are allowed for a key within the renewal_period.
   "key":
     "description": |-
-      Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.
+      Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported,
+      with 'endpoint' being the default if not specified.
   "renewal_period":
     "description": |-
       Renewal period field for a rate limit. Currently, only 'minute' is supported.
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitKey:
   "_":
-    "description": |-
-      Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.
     "enum":
       - |-
         user
@@ -2768,8 +2781,6 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitKey:
         endpoint
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitRenewalPeriod:
   "_":
-    "description": |-
-      Renewal period field for a rate limit. Currently, only 'minute' is supported.
     "enum":
       - |-
         minute
@@ -2780,26 +2791,43 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayUsageTrackingCo
 github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfig:
   "aws_access_key_id":
     "description": |-
-      The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.
+      The Databricks secret key reference for an AWS access key ID with
+      permissions to interact with Bedrock services. If you prefer to paste
+      your API key directly, see `aws_access_key_id_plaintext`. You must provide an API
+      key using one of the following fields: `aws_access_key_id` or
+      `aws_access_key_id_plaintext`.
   "aws_access_key_id_plaintext":
     "description": |-
-      An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.
+      An AWS access key ID with permissions to interact with Bedrock services
+      provided as a plaintext string. If you prefer to reference your key using
+      Databricks Secrets, see `aws_access_key_id`. You must provide an API key
+      using one of the following fields: `aws_access_key_id` or
+      `aws_access_key_id_plaintext`.
   "aws_region":
     "description": |-
       The AWS region to use. Bedrock has to be enabled there.
   "aws_secret_access_key":
     "description": |-
-      The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.
+      The Databricks secret key reference for an AWS secret access key paired
+      with the access key ID, with permissions to interact with Bedrock
+      services. If you prefer to paste your API key directly, see
+      `aws_secret_access_key_plaintext`. You must provide an API key using one
+      of the following fields: `aws_secret_access_key` or
+      `aws_secret_access_key_plaintext`.
   "aws_secret_access_key_plaintext":
     "description": |-
-      An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.
+      An AWS secret access key paired with the access key ID, with permissions
+      to interact with Bedrock services provided as a plaintext string. If you
+      prefer to reference your key using Databricks Secrets, see
+      `aws_secret_access_key`. You must provide an API key using one of the
+      following fields: `aws_secret_access_key` or
+      `aws_secret_access_key_plaintext`.
   "bedrock_provider":
     "description": |-
-      The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.
+      The underlying provider in Amazon Bedrock. Supported values (case
+      insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.
 github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfigBedrockProvider:
   "_":
-    "description": |-
-      The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.
     "enum":
       - |-
         anthropic
@@ -2812,10 +2840,16 @@ github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfigBedro
 github.com/databricks/databricks-sdk-go/service/serving.AnthropicConfig:
   "anthropic_api_key":
     "description": |-
-      The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.
+      The Databricks secret key reference for an Anthropic API key. If you
+      prefer to paste your API key directly, see `anthropic_api_key_plaintext`.
+      You must provide an API key using one of the following fields:
+      `anthropic_api_key` or `anthropic_api_key_plaintext`.
   "anthropic_api_key_plaintext":
     "description": |-
-      The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.
+      The Anthropic API key provided as a plaintext string. If you prefer to
+      reference your key using Databricks Secrets, see `anthropic_api_key`. You
+      must provide an API key using one of the following fields:
+      `anthropic_api_key` or `anthropic_api_key_plaintext`.
 github.com/databricks/databricks-sdk-go/service/serving.AutoCaptureConfigInput:
   "catalog_name":
     "description": |-
@@ -2831,42 +2865,58 @@ github.com/databricks/databricks-sdk-go/service/serving.AutoCaptureConfigInput:
       The prefix of the table in Unity Catalog. NOTE: On update, you cannot change the prefix name if the inference table is already enabled.
 github.com/databricks/databricks-sdk-go/service/serving.CohereConfig:
   "cohere_api_base":
-    "description": "This is an optional field to provide a customized base URL for the Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n"
+    "description": |-
+      This is an optional field to provide a customized base URL for the Cohere
+      API. If left unspecified, the standard Cohere base URL is used.
   "cohere_api_key":
     "description": |-
-      The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.
+      The Databricks secret key reference for a Cohere API key. If you prefer
+      to paste your API key directly, see `cohere_api_key_plaintext`. You must
+      provide an API key using one of the following fields: `cohere_api_key` or
+      `cohere_api_key_plaintext`.
   "cohere_api_key_plaintext":
     "description": |-
-      The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.
+      The Cohere API key provided as a plaintext string. If you prefer to
+      reference your key using Databricks Secrets, see `cohere_api_key`. You
+      must provide an API key using one of the following fields:
+      `cohere_api_key` or `cohere_api_key_plaintext`.
 github.com/databricks/databricks-sdk-go/service/serving.DatabricksModelServingConfig:
   "databricks_api_token":
-    "description": |
-      The Databricks secret key reference for a Databricks API token that corresponds to a user or service
-      principal with Can Query access to the model serving endpoint pointed to by this external model.
-      If you prefer to paste your API key directly, see `databricks_api_token_plaintext`.
-      You must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.
+    "description": |-
+      The Databricks secret key reference for a Databricks API token that
+      corresponds to a user or service principal with Can Query access to the
+      model serving endpoint pointed to by this external model. If you prefer
+      to paste your API key directly, see `databricks_api_token_plaintext`. You
+      must provide an API key using one of the following fields:
+      `databricks_api_token` or `databricks_api_token_plaintext`.
   "databricks_api_token_plaintext":
-    "description": |
-      The Databricks API token that corresponds to a user or service
-      principal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string.
-      If you prefer to reference your key using Databricks Secrets, see `databricks_api_token`.
-      You must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.
+    "description": |-
+      The Databricks API token that corresponds to a user or service principal
+      with Can Query access to the model serving endpoint pointed to by this
+      external model provided as a plaintext string. If you prefer to reference
+      your key using Databricks Secrets, see `databricks_api_token`. You must
+      provide an API key using one of the following fields:
+      `databricks_api_token` or `databricks_api_token_plaintext`.
   "databricks_workspace_url":
-    "description": |
-      The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model.
+    "description": |-
+      The URL of the Databricks workspace containing the model serving endpoint
+      pointed to by this external model.
 github.com/databricks/databricks-sdk-go/service/serving.EndpointCoreConfigInput:
   "auto_capture_config":
     "description": |-
       Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog.
+      Note: this field is deprecated for creating new provisioned throughput endpoints,
+      or updating existing provisioned throughput endpoints that never have inference table configured;
+      in these cases please use AI Gateway to manage inference tables.
   "served_entities":
     "description": |-
-      A list of served entities for the endpoint to serve. A serving endpoint can have up to 15 served entities.
+      The list of served entities under the serving endpoint config.
   "served_models":
     "description": |-
-      (Deprecated, use served_entities instead) A list of served models for the endpoint to serve. A serving endpoint can have up to 15 served models.
+      (Deprecated, use served_entities instead) The list of served models under the serving endpoint config.
   "traffic_config":
     "description": |-
-      The traffic config defining how invocations to the serving endpoint should be routed.
+      The traffic configuration associated with the serving endpoint config.
 github.com/databricks/databricks-sdk-go/service/serving.EndpointTag:
   "key":
     "description": |-
@@ -2903,17 +2953,13 @@ github.com/databricks/databricks-sdk-go/service/serving.ExternalModel:
     "description": |-
       PaLM Config. Only required if the provider is 'palm'.
   "provider":
-    "description": |
-      The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',
-      'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.",
+    "description": |-
+      The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic', 'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.
   "task":
     "description": |-
       The task type of the external model.
 github.com/databricks/databricks-sdk-go/service/serving.ExternalModelProvider:
   "_":
-    "description": |
-      The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',
-      'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.",
     "enum":
       - |-
         ai21labs
@@ -2934,70 +2980,114 @@ github.com/databricks/databricks-sdk-go/service/serving.ExternalModelProvider:
 github.com/databricks/databricks-sdk-go/service/serving.GoogleCloudVertexAiConfig:
   "private_key":
     "description": |-
-      The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`
+      The Databricks secret key reference for a private key for the service
+      account which has access to the Google Cloud Vertex AI Service. See [Best
+      practices for managing service account keys]. If you prefer to paste your
+      API key directly, see `private_key_plaintext`. You must provide an API
+      key using one of the following fields: `private_key` or
+      `private_key_plaintext`
+
+      [Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys
   "private_key_plaintext":
     "description": |-
-      The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`.
+      The private key for the service account which has access to the Google
+      Cloud Vertex AI Service provided as a plaintext secret. See [Best
+      practices for managing service account keys]. If you prefer to reference
+      your key using Databricks Secrets, see `private_key`. You must provide an
+      API key using one of the following fields: `private_key` or
+      `private_key_plaintext`.
+
+      [Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys
   "project_id":
     "description": |-
-      This is the Google Cloud project id that the service account is associated with.
+      This is the Google Cloud project id that the service account is
+      associated with.
   "region":
     "description": |-
-      This is the region for the Google Cloud Vertex AI Service. See [supported regions](https://cloud.google.com/vertex-ai/docs/general/locations) for more details. Some models are only available in specific regions.
+      This is the region for the Google Cloud Vertex AI Service. See [supported
+      regions] for more details. Some models are only available in specific
+      regions.
+
+      [supported regions]: https://cloud.google.com/vertex-ai/docs/general/locations
 github.com/databricks/databricks-sdk-go/service/serving.OpenAiConfig:
+  "_":
+    "description": |-
+      Configs needed to create an OpenAI model route.
   "microsoft_entra_client_id":
-    "description": |
-      This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.
+    "description": |-
+      This field is only required for Azure AD OpenAI and is the Microsoft
+      Entra Client ID.
   "microsoft_entra_client_secret":
-    "description": |
-      The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication.
-      If you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`.
-      You must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.
+    "description": |-
+      The Databricks secret key reference for a client secret used for
+      Microsoft Entra ID authentication. If you prefer to paste your client
+      secret directly, see `microsoft_entra_client_secret_plaintext`. You must
+      provide an API key using one of the following fields:
+      `microsoft_entra_client_secret` or
+      `microsoft_entra_client_secret_plaintext`.
   "microsoft_entra_client_secret_plaintext":
-    "description": |
-      The client secret used for Microsoft Entra ID authentication provided as a plaintext string.
-      If you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`.
-      You must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.
+    "description": |-
+      The client secret used for Microsoft Entra ID authentication provided as
+      a plaintext string. If you prefer to reference your key using Databricks
+      Secrets, see `microsoft_entra_client_secret`. You must provide an API key
+      using one of the following fields: `microsoft_entra_client_secret` or
+      `microsoft_entra_client_secret_plaintext`.
   "microsoft_entra_tenant_id":
-    "description": |
-      This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.
+    "description": |-
+      This field is only required for Azure AD OpenAI and is the Microsoft
+      Entra Tenant ID.
   "openai_api_base":
-    "description": |
-      This is a field to provide a customized base URl for the OpenAI API.
-      For Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service
-      provided by Azure.
-      For other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used.
+    "description": |-
+      This is a field to provide a customized base URl for the OpenAI API. For
+      Azure OpenAI, this field is required, and is the base URL for the Azure
+      OpenAI API service provided by Azure. For other OpenAI API types, this
+      field is optional, and if left unspecified, the standard OpenAI base URL
+      is used.
   "openai_api_key":
     "description": |-
-      The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`.
+      The Databricks secret key reference for an OpenAI API key using the
+      OpenAI or Azure service. If you prefer to paste your API key directly,
+      see `openai_api_key_plaintext`. You must provide an API key using one of
+      the following fields: `openai_api_key` or `openai_api_key_plaintext`.
   "openai_api_key_plaintext":
     "description": |-
-      The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`.
+      The OpenAI API key using the OpenAI or Azure service provided as a
+      plaintext string. If you prefer to reference your key using Databricks
+      Secrets, see `openai_api_key`. You must provide an API key using one of
+      the following fields: `openai_api_key` or `openai_api_key_plaintext`.
   "openai_api_type":
-    "description": |
-      This is an optional field to specify the type of OpenAI API to use.
-      For Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security
-      access validation protocol. For access token validation, use azure. For authentication using Azure Active
+    "description": |-
+      This is an optional field to specify the type of OpenAI API to use. For
+      Azure OpenAI, this field is required, and adjust this parameter to
+      represent the preferred security access validation protocol. For access
+      token validation, use azure. For authentication using Azure Active
       Directory (Azure AD) use, azuread.
   "openai_api_version":
-    "description": |
-      This is an optional field to specify the OpenAI API version.
-      For Azure OpenAI, this field is required, and is the version of the Azure OpenAI service to
-      utilize, specified by a date.
+    "description": |-
+      This is an optional field to specify the OpenAI API version. For Azure
+      OpenAI, this field is required, and is the version of the Azure OpenAI
+      service to utilize, specified by a date.
   "openai_deployment_name":
-    "description": |
-      This field is only required for Azure OpenAI and is the name of the deployment resource for the
-      Azure OpenAI service.
+    "description": |-
+      This field is only required for Azure OpenAI and is the name of the
+      deployment resource for the Azure OpenAI service.
   "openai_organization":
-    "description": |
-      This is an optional field to specify the organization in OpenAI or Azure OpenAI.
+    "description": |-
+      This is an optional field to specify the organization in OpenAI or Azure
+      OpenAI.
 github.com/databricks/databricks-sdk-go/service/serving.PaLmConfig:
   "palm_api_key":
     "description": |-
-      The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`.
+      The Databricks secret key reference for a PaLM API key. If you prefer to
+      paste your API key directly, see `palm_api_key_plaintext`. You must
+      provide an API key using one of the following fields: `palm_api_key` or
+      `palm_api_key_plaintext`.
   "palm_api_key_plaintext":
     "description": |-
-      The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`.
+      The PaLM API key provided as a plaintext string. If you prefer to
+      reference your key using Databricks Secrets, see `palm_api_key`. You must
+      provide an API key using one of the following fields: `palm_api_key` or
+      `palm_api_key_plaintext`.
 github.com/databricks/databricks-sdk-go/service/serving.RateLimit:
   "calls":
     "description": |-
@@ -3010,8 +3100,6 @@ github.com/databricks/databricks-sdk-go/service/serving.RateLimit:
       Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported.
 github.com/databricks/databricks-sdk-go/service/serving.RateLimitKey:
   "_":
-    "description": |-
-      Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.
     "enum":
       - |-
         user
@@ -3019,8 +3107,6 @@ github.com/databricks/databricks-sdk-go/service/serving.RateLimitKey:
         endpoint
 github.com/databricks/databricks-sdk-go/service/serving.RateLimitRenewalPeriod:
   "_":
-    "description": |-
-      Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported.
     "enum":
       - |-
         minute
@@ -3033,21 +3119,15 @@ github.com/databricks/databricks-sdk-go/service/serving.Route:
       The percentage of endpoint traffic to send to this route. It must be an integer between 0 and 100 inclusive.
 github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput:
   "entity_name":
-    "description": |
-      The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC),
-      or a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of
-      __catalog_name__.__schema_name__.__model_name__.
-  "entity_version":
     "description": |-
-      The version of the model in Databricks Model Registry to be served or empty if the entity is a FEATURE_SPEC.
+      The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC), or a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of **catalog_name.schema_name.model_name**.
+  "entity_version": {}
   "environment_vars":
-    "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity.\nNote: this is an experimental feature and subject to change. \nExample entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`"
+    "description": |-
+      An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{"OPENAI_API_KEY": "{{secrets/my_scope/my_key}}", "DATABRICKS_TOKEN": "{{secrets/my_scope2/my_key2}}"}`
   "external_model":
-    "description": |
-      The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)
-      can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model,
-      it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later.
-      The task type of all external models within an endpoint must be the same.
+    "description": |-
+      The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same.
   "instance_profile_arn":
     "description": |-
       ARN of the instance profile that the served entity uses to access AWS resources.
@@ -3058,68 +3138,46 @@ github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput:
     "description": |-
       The minimum tokens per second that the endpoint can scale down to.
   "name":
-    "description": |
-      The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores.
-      If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other
-      entities, it defaults to <entity-name>-<entity-version>.
+    "description": |-
+      The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.
   "scale_to_zero_enabled":
     "description": |-
       Whether the compute resources for the served entity should scale down to zero.
   "workload_size":
-    "description": |
-      The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between.
-      A single unit of provisioned concurrency can process one request at a time.
-      Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency).
-      If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.
+    "description": |-
+      The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.
   "workload_type":
-    "description": |
-      The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is
-      "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.
-      See the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).
+    "description": |-
+      The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).
 github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput:
   "environment_vars":
-    "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this model.\nNote: this is an experimental feature and subject to change. \nExample model environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`"
+    "description": |-
+      An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{"OPENAI_API_KEY": "{{secrets/my_scope/my_key}}", "DATABRICKS_TOKEN": "{{secrets/my_scope2/my_key2}}"}`
   "instance_profile_arn":
     "description": |-
-      ARN of the instance profile that the served model will use to access AWS resources.
+      ARN of the instance profile that the served entity uses to access AWS resources.
   "max_provisioned_throughput":
     "description": |-
       The maximum tokens per second that the endpoint can scale up to.
   "min_provisioned_throughput":
     "description": |-
       The minimum tokens per second that the endpoint can scale down to.
-  "model_name":
-    "description": |
-      The name of the model in Databricks Model Registry to be served or if the model resides in Unity Catalog, the full name of model,
-      in the form of __catalog_name__.__schema_name__.__model_name__.
-  "model_version":
-    "description": |-
-      The version of the model in Databricks Model Registry or Unity Catalog to be served.
+  "model_name": {}
+  "model_version": {}
   "name":
-    "description": |
-      The name of a served model. It must be unique across an endpoint. If not specified, this field will default to <model-name>-<model-version>.
-      A served model name can consist of alphanumeric characters, dashes, and underscores.
+    "description": |-
+      The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.
   "scale_to_zero_enabled":
     "description": |-
-      Whether the compute resources for the served model should scale down to zero.
+      Whether the compute resources for the served entity should scale down to zero.
   "workload_size":
-    "description": |
-      The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.
-      A single unit of provisioned concurrency can process one request at a time.
-      Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency).
-      If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.
+    "description": |-
+      The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.
   "workload_type":
-    "description": |
-      The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is
-      "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.
-      See the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).
+    "description": |-
+      The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).
 github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadSize:
   "_":
-    "description": |
-      The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.
-      A single unit of provisioned concurrency can process one request at a time.
-      Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency).
-      If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.
     "enum":
       - |-
         Small
@@ -3129,17 +3187,26 @@ github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkload
         Large
 github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadType:
   "_":
-    "description": |
-      The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is
-      "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.
-      See the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).
     "enum":
       - |-
         CPU
+      - |-
+        GPU_MEDIUM
       - |-
         GPU_SMALL
+      - |-
+        GPU_LARGE
+      - |-
+        MULTIGPU_MEDIUM
+github.com/databricks/databricks-sdk-go/service/serving.ServingModelWorkloadType:
+  "_":
+    "enum":
+      - |-
+        CPU
       - |-
         GPU_MEDIUM
+      - |-
+        GPU_SMALL
       - |-
         GPU_LARGE
       - |-
diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml
index 120a12543..323432fa3 100644
--- a/bundle/internal/schema/annotations_openapi_overrides.yml
+++ b/bundle/internal/schema/annotations_openapi_overrides.yml
@@ -197,3 +197,14 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger:
   "manual":
     "description": |-
       PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput:
+  "entity_version":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput:
+  "model_name":
+    "description": |-
+      PLACEHOLDER
+  "model_version":
+    "description": |-
+      PLACEHOLDER
diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json
index 4a3b56814..17a621ba0 100644
--- a/bundle/schema/jsonschema.json
+++ b/bundle/schema/jsonschema.json
@@ -546,7 +546,7 @@
                     "type": "object",
                     "properties": {
                       "ai_gateway": {
-                        "description": "The AI Gateway configuration for the serving endpoint. NOTE: only external model endpoints are supported as of now.",
+                        "description": "The AI Gateway configuration for the serving endpoint. NOTE: Only external model and provisioned throughput endpoints are currently supported.",
                         "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayConfig"
                       },
                       "config": {
@@ -554,7 +554,7 @@
                         "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.EndpointCoreConfigInput"
                       },
                       "name": {
-                        "description": "The name of the serving endpoint. This field is required and must be unique across a Databricks workspace.\nAn endpoint name can consist of alphanumeric characters, dashes, and underscores.\n",
+                        "description": "The name of the serving endpoint. This field is required and must be unique across a Databricks workspace.\nAn endpoint name can consist of alphanumeric characters, dashes, and underscores.",
                         "$ref": "#/$defs/string"
                       },
                       "permissions": {
@@ -575,7 +575,6 @@
                     },
                     "additionalProperties": false,
                     "required": [
-                      "config",
                       "name"
                     ]
                   },
@@ -4142,6 +4141,10 @@
                     "parameters": {
                       "description": "Parameters passed to the main method.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.",
                       "$ref": "#/$defs/slice/string"
+                    },
+                    "run_as_repl": {
+                      "description": "Deprecated. A value of `false` is no longer supported.",
+                      "$ref": "#/$defs/bool"
                     }
                   },
                   "additionalProperties": false
@@ -5502,11 +5505,11 @@
                   "type": "object",
                   "properties": {
                     "ai21labs_api_key": {
-                      "description": "The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`.",
+                      "description": "The Databricks secret key reference for an AI21 Labs API key. If you\nprefer to paste your API key directly, see `ai21labs_api_key_plaintext`.\nYou must provide an API key using one of the following fields:\n`ai21labs_api_key` or `ai21labs_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "ai21labs_api_key_plaintext": {
-                      "description": "An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`.",
+                      "description": "An AI21 Labs API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `ai21labs_api_key`. You\nmust provide an API key using one of the following fields:\n`ai21labs_api_key` or `ai21labs_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -5528,7 +5531,7 @@
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrails"
                     },
                     "inference_table_config": {
-                      "description": "Configuration for payload logging using inference tables. Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.",
+                      "description": "Configuration for payload logging using inference tables.\nUse these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayInferenceTableConfig"
                     },
                     "rate_limits": {
@@ -5536,7 +5539,7 @@
                       "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit"
                     },
                     "usage_tracking_config": {
-                      "description": "Configuration to enable usage tracking using system tables. These tables allow you to monitor operational usage on endpoints and their associated costs.",
+                      "description": "Configuration to enable usage tracking using system tables.\nThese tables allow you to monitor operational usage on endpoints and their associated costs.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayUsageTrackingConfig"
                     }
                   },
@@ -5554,7 +5557,7 @@
                   "type": "object",
                   "properties": {
                     "invalid_keywords": {
-                      "description": "List of invalid keywords. AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.",
+                      "description": "List of invalid keywords.\nAI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.",
                       "$ref": "#/$defs/slice/string"
                     },
                     "pii": {
@@ -5566,7 +5569,7 @@
                       "$ref": "#/$defs/bool"
                     },
                     "valid_topics": {
-                      "description": "The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics.",
+                      "description": "The list of allowed topics.\nGiven a chat request, this guardrail flags the request if its topic is not in the allowed topics.",
                       "$ref": "#/$defs/slice/string"
                     }
                   },
@@ -5584,14 +5587,11 @@
                   "type": "object",
                   "properties": {
                     "behavior": {
-                      "description": "Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.",
+                      "description": "Configuration for input guardrail filters.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehaviorBehavior"
                     }
                   },
-                  "additionalProperties": false,
-                  "required": [
-                    "behavior"
-                  ]
+                  "additionalProperties": false
                 },
                 {
                   "type": "string",
@@ -5603,7 +5603,6 @@
               "oneOf": [
                 {
                   "type": "string",
-                  "description": "Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.",
                   "enum": [
                     "NONE",
                     "BLOCK"
@@ -5643,7 +5642,7 @@
                   "type": "object",
                   "properties": {
                     "catalog_name": {
-                      "description": "The name of the catalog in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the catalog name.",
+                      "description": "The name of the catalog in Unity Catalog. Required when enabling inference tables.\nNOTE: On update, you have to disable inference table first in order to change the catalog name.",
                       "$ref": "#/$defs/string"
                     },
                     "enabled": {
@@ -5651,11 +5650,11 @@
                       "$ref": "#/$defs/bool"
                     },
                     "schema_name": {
-                      "description": "The name of the schema in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the schema name.",
+                      "description": "The name of the schema in Unity Catalog. Required when enabling inference tables.\nNOTE: On update, you have to disable inference table first in order to change the schema name.",
                       "$ref": "#/$defs/string"
                     },
                     "table_name_prefix": {
-                      "description": "The prefix of the table in Unity Catalog. NOTE: On update, you have to disable inference table first in order to change the prefix name.",
+                      "description": "The prefix of the table in Unity Catalog.\nNOTE: On update, you have to disable inference table first in order to change the prefix name.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -5674,10 +5673,10 @@
                   "properties": {
                     "calls": {
                       "description": "Used to specify how many calls are allowed for a key within the renewal_period.",
-                      "$ref": "#/$defs/int"
+                      "$ref": "#/$defs/int64"
                     },
                     "key": {
-                      "description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.",
+                      "description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported,\nwith 'endpoint' being the default if not specified.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitKey"
                     },
                     "renewal_period": {
@@ -5701,7 +5700,6 @@
               "oneOf": [
                 {
                   "type": "string",
-                  "description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.",
                   "enum": [
                     "user",
                     "endpoint"
@@ -5717,7 +5715,6 @@
               "oneOf": [
                 {
                   "type": "string",
-                  "description": "Renewal period field for a rate limit. Currently, only 'minute' is supported.",
                   "enum": [
                     "minute"
                   ]
@@ -5752,11 +5749,11 @@
                   "type": "object",
                   "properties": {
                     "aws_access_key_id": {
-                      "description": "The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.",
+                      "description": "The Databricks secret key reference for an AWS access key ID with\npermissions to interact with Bedrock services. If you prefer to paste\nyour API key directly, see `aws_access_key_id_plaintext`. You must provide an API\nkey using one of the following fields: `aws_access_key_id` or\n`aws_access_key_id_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "aws_access_key_id_plaintext": {
-                      "description": "An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.",
+                      "description": "An AWS access key ID with permissions to interact with Bedrock services\nprovided as a plaintext string. If you prefer to reference your key using\nDatabricks Secrets, see `aws_access_key_id`. You must provide an API key\nusing one of the following fields: `aws_access_key_id` or\n`aws_access_key_id_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "aws_region": {
@@ -5764,15 +5761,15 @@
                       "$ref": "#/$defs/string"
                     },
                     "aws_secret_access_key": {
-                      "description": "The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.",
+                      "description": "The Databricks secret key reference for an AWS secret access key paired\nwith the access key ID, with permissions to interact with Bedrock\nservices. If you prefer to paste your API key directly, see\n`aws_secret_access_key_plaintext`. You must provide an API key using one\nof the following fields: `aws_secret_access_key` or\n`aws_secret_access_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "aws_secret_access_key_plaintext": {
-                      "description": "An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.",
+                      "description": "An AWS secret access key paired with the access key ID, with permissions\nto interact with Bedrock services provided as a plaintext string. If you\nprefer to reference your key using Databricks Secrets, see\n`aws_secret_access_key`. You must provide an API key using one of the\nfollowing fields: `aws_secret_access_key` or\n`aws_secret_access_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "bedrock_provider": {
-                      "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.",
+                      "description": "The underlying provider in Amazon Bedrock. Supported values (case\ninsensitive) include: Anthropic, Cohere, AI21Labs, Amazon.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfigBedrockProvider"
                     }
                   },
@@ -5792,7 +5789,6 @@
               "oneOf": [
                 {
                   "type": "string",
-                  "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.",
                   "enum": [
                     "anthropic",
                     "cohere",
@@ -5812,11 +5808,11 @@
                   "type": "object",
                   "properties": {
                     "anthropic_api_key": {
-                      "description": "The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.",
+                      "description": "The Databricks secret key reference for an Anthropic API key. If you\nprefer to paste your API key directly, see `anthropic_api_key_plaintext`.\nYou must provide an API key using one of the following fields:\n`anthropic_api_key` or `anthropic_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "anthropic_api_key_plaintext": {
-                      "description": "The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.",
+                      "description": "The Anthropic API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `anthropic_api_key`. You\nmust provide an API key using one of the following fields:\n`anthropic_api_key` or `anthropic_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -5864,15 +5860,15 @@
                   "type": "object",
                   "properties": {
                     "cohere_api_base": {
-                      "description": "This is an optional field to provide a customized base URL for the Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n",
+                      "description": "This is an optional field to provide a customized base URL for the Cohere\nAPI. If left unspecified, the standard Cohere base URL is used.",
                       "$ref": "#/$defs/string"
                     },
                     "cohere_api_key": {
-                      "description": "The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.",
+                      "description": "The Databricks secret key reference for a Cohere API key. If you prefer\nto paste your API key directly, see `cohere_api_key_plaintext`. You must\nprovide an API key using one of the following fields: `cohere_api_key` or\n`cohere_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "cohere_api_key_plaintext": {
-                      "description": "The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.",
+                      "description": "The Cohere API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `cohere_api_key`. You\nmust provide an API key using one of the following fields:\n`cohere_api_key` or `cohere_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -5890,15 +5886,15 @@
                   "type": "object",
                   "properties": {
                     "databricks_api_token": {
-                      "description": "The Databricks secret key reference for a Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model.\nIf you prefer to paste your API key directly, see `databricks_api_token_plaintext`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n",
+                      "description": "The Databricks secret key reference for a Databricks API token that\ncorresponds to a user or service principal with Can Query access to the\nmodel serving endpoint pointed to by this external model. If you prefer\nto paste your API key directly, see `databricks_api_token_plaintext`. You\nmust provide an API key using one of the following fields:\n`databricks_api_token` or `databricks_api_token_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "databricks_api_token_plaintext": {
-                      "description": "The Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `databricks_api_token`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n",
+                      "description": "The Databricks API token that corresponds to a user or service principal\nwith Can Query access to the model serving endpoint pointed to by this\nexternal model provided as a plaintext string. If you prefer to reference\nyour key using Databricks Secrets, see `databricks_api_token`. You must\nprovide an API key using one of the following fields:\n`databricks_api_token` or `databricks_api_token_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "databricks_workspace_url": {
-                      "description": "The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model.\n",
+                      "description": "The URL of the Databricks workspace containing the model serving endpoint\npointed to by this external model.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -5919,19 +5915,19 @@
                   "type": "object",
                   "properties": {
                     "auto_capture_config": {
-                      "description": "Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog.",
+                      "description": "Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog.\nNote: this field is deprecated for creating new provisioned throughput endpoints,\nor updating existing provisioned throughput endpoints that never have inference table configured;\nin these cases please use AI Gateway to manage inference tables.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AutoCaptureConfigInput"
                     },
                     "served_entities": {
-                      "description": "A list of served entities for the endpoint to serve. A serving endpoint can have up to 15 served entities.",
+                      "description": "The list of served entities under the serving endpoint config.",
                       "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput"
                     },
                     "served_models": {
-                      "description": "(Deprecated, use served_entities instead) A list of served models for the endpoint to serve. A serving endpoint can have up to 15 served models.",
+                      "description": "(Deprecated, use served_entities instead) The list of served models under the serving endpoint config.",
                       "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput"
                     },
                     "traffic_config": {
-                      "description": "The traffic config defining how invocations to the serving endpoint should be routed.",
+                      "description": "The traffic configuration associated with the serving endpoint config.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.TrafficConfig"
                     }
                   },
@@ -6010,7 +6006,7 @@
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.PaLmConfig"
                     },
                     "provider": {
-                      "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n",
+                      "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic', 'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ExternalModelProvider"
                     },
                     "task": {
@@ -6035,7 +6031,6 @@
               "oneOf": [
                 {
                   "type": "string",
-                  "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n",
                   "enum": [
                     "ai21labs",
                     "anthropic",
@@ -6059,23 +6054,27 @@
                   "type": "object",
                   "properties": {
                     "private_key": {
-                      "description": "The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`",
+                      "description": "The Databricks secret key reference for a private key for the service\naccount which has access to the Google Cloud Vertex AI Service. See [Best\npractices for managing service account keys]. If you prefer to paste your\nAPI key directly, see `private_key_plaintext`. You must provide an API\nkey using one of the following fields: `private_key` or\n`private_key_plaintext`\n\n[Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys",
                       "$ref": "#/$defs/string"
                     },
                     "private_key_plaintext": {
-                      "description": "The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`.",
+                      "description": "The private key for the service account which has access to the Google\nCloud Vertex AI Service provided as a plaintext secret. See [Best\npractices for managing service account keys]. If you prefer to reference\nyour key using Databricks Secrets, see `private_key`. You must provide an\nAPI key using one of the following fields: `private_key` or\n`private_key_plaintext`.\n\n[Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys",
                       "$ref": "#/$defs/string"
                     },
                     "project_id": {
-                      "description": "This is the Google Cloud project id that the service account is associated with.",
+                      "description": "This is the Google Cloud project id that the service account is\nassociated with.",
                       "$ref": "#/$defs/string"
                     },
                     "region": {
-                      "description": "This is the region for the Google Cloud Vertex AI Service. See [supported regions](https://cloud.google.com/vertex-ai/docs/general/locations) for more details. Some models are only available in specific regions.",
+                      "description": "This is the region for the Google Cloud Vertex AI Service. See [supported\nregions] for more details. Some models are only available in specific\nregions.\n\n[supported regions]: https://cloud.google.com/vertex-ai/docs/general/locations",
                       "$ref": "#/$defs/string"
                     }
                   },
-                  "additionalProperties": false
+                  "additionalProperties": false,
+                  "required": [
+                    "project_id",
+                    "region"
+                  ]
                 },
                 {
                   "type": "string",
@@ -6087,49 +6086,50 @@
               "oneOf": [
                 {
                   "type": "object",
+                  "description": "Configs needed to create an OpenAI model route.",
                   "properties": {
                     "microsoft_entra_client_id": {
-                      "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.\n",
+                      "description": "This field is only required for Azure AD OpenAI and is the Microsoft\nEntra Client ID.",
                       "$ref": "#/$defs/string"
                     },
                     "microsoft_entra_client_secret": {
-                      "description": "The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication.\nIf you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n",
+                      "description": "The Databricks secret key reference for a client secret used for\nMicrosoft Entra ID authentication. If you prefer to paste your client\nsecret directly, see `microsoft_entra_client_secret_plaintext`. You must\nprovide an API key using one of the following fields:\n`microsoft_entra_client_secret` or\n`microsoft_entra_client_secret_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "microsoft_entra_client_secret_plaintext": {
-                      "description": "The client secret used for Microsoft Entra ID authentication provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n",
+                      "description": "The client secret used for Microsoft Entra ID authentication provided as\na plaintext string. If you prefer to reference your key using Databricks\nSecrets, see `microsoft_entra_client_secret`. You must provide an API key\nusing one of the following fields: `microsoft_entra_client_secret` or\n`microsoft_entra_client_secret_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "microsoft_entra_tenant_id": {
-                      "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.\n",
+                      "description": "This field is only required for Azure AD OpenAI and is the Microsoft\nEntra Tenant ID.",
                       "$ref": "#/$defs/string"
                     },
                     "openai_api_base": {
-                      "description": "This is a field to provide a customized base URl for the OpenAI API.\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\nFor other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used.\n",
+                      "description": "This is a field to provide a customized base URl for the OpenAI API. For\nAzure OpenAI, this field is required, and is the base URL for the Azure\nOpenAI API service provided by Azure. For other OpenAI API types, this\nfield is optional, and if left unspecified, the standard OpenAI base URL\nis used.",
                       "$ref": "#/$defs/string"
                     },
                     "openai_api_key": {
-                      "description": "The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`.",
+                      "description": "The Databricks secret key reference for an OpenAI API key using the\nOpenAI or Azure service. If you prefer to paste your API key directly,\nsee `openai_api_key_plaintext`. You must provide an API key using one of\nthe following fields: `openai_api_key` or `openai_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "openai_api_key_plaintext": {
-                      "description": "The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`.",
+                      "description": "The OpenAI API key using the OpenAI or Azure service provided as a\nplaintext string. If you prefer to reference your key using Databricks\nSecrets, see `openai_api_key`. You must provide an API key using one of\nthe following fields: `openai_api_key` or `openai_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "openai_api_type": {
-                      "description": "This is an optional field to specify the type of OpenAI API to use.\nFor Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security\naccess validation protocol. For access token validation, use azure. For authentication using Azure Active\nDirectory (Azure AD) use, azuread.\n",
+                      "description": "This is an optional field to specify the type of OpenAI API to use. For\nAzure OpenAI, this field is required, and adjust this parameter to\nrepresent the preferred security access validation protocol. For access\ntoken validation, use azure. For authentication using Azure Active\nDirectory (Azure AD) use, azuread.",
                       "$ref": "#/$defs/string"
                     },
                     "openai_api_version": {
-                      "description": "This is an optional field to specify the OpenAI API version.\nFor Azure OpenAI, this field is required, and is the version of the Azure OpenAI service to\nutilize, specified by a date.\n",
+                      "description": "This is an optional field to specify the OpenAI API version. For Azure\nOpenAI, this field is required, and is the version of the Azure OpenAI\nservice to utilize, specified by a date.",
                       "$ref": "#/$defs/string"
                     },
                     "openai_deployment_name": {
-                      "description": "This field is only required for Azure OpenAI and is the name of the deployment resource for the\nAzure OpenAI service.\n",
+                      "description": "This field is only required for Azure OpenAI and is the name of the\ndeployment resource for the Azure OpenAI service.",
                       "$ref": "#/$defs/string"
                     },
                     "openai_organization": {
-                      "description": "This is an optional field to specify the organization in OpenAI or Azure OpenAI.\n",
+                      "description": "This is an optional field to specify the organization in OpenAI or Azure\nOpenAI.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -6147,11 +6147,11 @@
                   "type": "object",
                   "properties": {
                     "palm_api_key": {
-                      "description": "The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`.",
+                      "description": "The Databricks secret key reference for a PaLM API key. If you prefer to\npaste your API key directly, see `palm_api_key_plaintext`. You must\nprovide an API key using one of the following fields: `palm_api_key` or\n`palm_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "palm_api_key_plaintext": {
-                      "description": "The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`.",
+                      "description": "The PaLM API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `palm_api_key`. You must\nprovide an API key using one of the following fields: `palm_api_key` or\n`palm_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -6170,7 +6170,7 @@
                   "properties": {
                     "calls": {
                       "description": "Used to specify how many calls are allowed for a key within the renewal_period.",
-                      "$ref": "#/$defs/int"
+                      "$ref": "#/$defs/int64"
                     },
                     "key": {
                       "description": "Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.",
@@ -6197,7 +6197,6 @@
               "oneOf": [
                 {
                   "type": "string",
-                  "description": "Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.",
                   "enum": [
                     "user",
                     "endpoint"
@@ -6213,7 +6212,6 @@
               "oneOf": [
                 {
                   "type": "string",
-                  "description": "Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported.",
                   "enum": [
                     "minute"
                   ]
@@ -6256,19 +6254,18 @@
                   "type": "object",
                   "properties": {
                     "entity_name": {
-                      "description": "The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC),\nor a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of\n__catalog_name__.__schema_name__.__model_name__.\n",
+                      "description": "The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC), or a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of **catalog_name.schema_name.model_name**.",
                       "$ref": "#/$defs/string"
                     },
                     "entity_version": {
-                      "description": "The version of the model in Databricks Model Registry to be served or empty if the entity is a FEATURE_SPEC.",
                       "$ref": "#/$defs/string"
                     },
                     "environment_vars": {
-                      "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity.\nNote: this is an experimental feature and subject to change. \nExample entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`",
+                      "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`",
                       "$ref": "#/$defs/map/string"
                     },
                     "external_model": {
-                      "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)\ncan be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model,\nit cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later.\nThe task type of all external models within an endpoint must be the same.\n",
+                      "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ExternalModel"
                     },
                     "instance_profile_arn": {
@@ -6284,7 +6281,7 @@
                       "$ref": "#/$defs/int"
                     },
                     "name": {
-                      "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores.\nIf not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other\nentities, it defaults to \u003centity-name\u003e-\u003centity-version\u003e.\n",
+                      "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.",
                       "$ref": "#/$defs/string"
                     },
                     "scale_to_zero_enabled": {
@@ -6292,12 +6289,12 @@
                       "$ref": "#/$defs/bool"
                     },
                     "workload_size": {
-                      "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.\n",
+                      "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.",
                       "$ref": "#/$defs/string"
                     },
                     "workload_type": {
-                      "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n",
-                      "$ref": "#/$defs/string"
+                      "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is \"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).",
+                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServingModelWorkloadType"
                     }
                   },
                   "additionalProperties": false
@@ -6314,11 +6311,11 @@
                   "type": "object",
                   "properties": {
                     "environment_vars": {
-                      "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this model.\nNote: this is an experimental feature and subject to change. \nExample model environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`",
+                      "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`",
                       "$ref": "#/$defs/map/string"
                     },
                     "instance_profile_arn": {
-                      "description": "ARN of the instance profile that the served model will use to access AWS resources.",
+                      "description": "ARN of the instance profile that the served entity uses to access AWS resources.",
                       "$ref": "#/$defs/string"
                     },
                     "max_provisioned_throughput": {
@@ -6330,27 +6327,25 @@
                       "$ref": "#/$defs/int"
                     },
                     "model_name": {
-                      "description": "The name of the model in Databricks Model Registry to be served or if the model resides in Unity Catalog, the full name of model,\nin the form of __catalog_name__.__schema_name__.__model_name__.\n",
                       "$ref": "#/$defs/string"
                     },
                     "model_version": {
-                      "description": "The version of the model in Databricks Model Registry or Unity Catalog to be served.",
                       "$ref": "#/$defs/string"
                     },
                     "name": {
-                      "description": "The name of a served model. It must be unique across an endpoint. If not specified, this field will default to \u003cmodel-name\u003e-\u003cmodel-version\u003e.\nA served model name can consist of alphanumeric characters, dashes, and underscores.\n",
+                      "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.",
                       "$ref": "#/$defs/string"
                     },
                     "scale_to_zero_enabled": {
-                      "description": "Whether the compute resources for the served model should scale down to zero.",
+                      "description": "Whether the compute resources for the served entity should scale down to zero.",
                       "$ref": "#/$defs/bool"
                     },
                     "workload_size": {
-                      "description": "The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.\n",
+                      "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadSize"
                     },
                     "workload_type": {
-                      "description": "The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n",
+                      "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is \"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadType"
                     }
                   },
@@ -6371,7 +6366,6 @@
               "oneOf": [
                 {
                   "type": "string",
-                  "description": "The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.\n",
                   "enum": [
                     "Small",
                     "Medium",
@@ -6388,11 +6382,28 @@
               "oneOf": [
                 {
                   "type": "string",
-                  "description": "The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n",
                   "enum": [
                     "CPU",
-                    "GPU_SMALL",
                     "GPU_MEDIUM",
+                    "GPU_SMALL",
+                    "GPU_LARGE",
+                    "MULTIGPU_MEDIUM"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
+              ]
+            },
+            "serving.ServingModelWorkloadType": {
+              "oneOf": [
+                {
+                  "type": "string",
+                  "enum": [
+                    "CPU",
+                    "GPU_MEDIUM",
+                    "GPU_SMALL",
                     "GPU_LARGE",
                     "MULTIGPU_MEDIUM"
                   ]
diff --git a/cmd/account/custom-app-integration/custom-app-integration.go b/cmd/account/custom-app-integration/custom-app-integration.go
index 1eec1018e..43e458bc6 100755
--- a/cmd/account/custom-app-integration/custom-app-integration.go
+++ b/cmd/account/custom-app-integration/custom-app-integration.go
@@ -307,6 +307,7 @@ func newUpdate() *cobra.Command {
 	cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
 	// TODO: array: redirect_urls
+	// TODO: array: scopes
 	// TODO: complex arg: token_access_policy
 
 	cmd.Use = "update INTEGRATION_ID"
diff --git a/cmd/api/api.go b/cmd/api/api.go
index c3a3eb0b6..fad8a026f 100644
--- a/cmd/api/api.go
+++ b/cmd/api/api.go
@@ -62,7 +62,7 @@ func makeCommand(method string) *cobra.Command {
 
 			var response any
 			headers := map[string]string{"Content-Type": "application/json"}
-			err = api.Do(cmd.Context(), method, path, headers, request, &response)
+			err = api.Do(cmd.Context(), method, path, headers, nil, request, &response)
 			if err != nil {
 				return err
 			}
diff --git a/cmd/workspace/access-control/access-control.go b/cmd/workspace/access-control/access-control.go
new file mode 100755
index 000000000..7668265fb
--- /dev/null
+++ b/cmd/workspace/access-control/access-control.go
@@ -0,0 +1,109 @@
+// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT.
+
+package access_control
+
+import (
+	"fmt"
+
+	"github.com/databricks/cli/cmd/root"
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/flags"
+	"github.com/databricks/databricks-sdk-go/service/iam"
+	"github.com/spf13/cobra"
+)
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var cmdOverrides []func(*cobra.Command)
+
+func New() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "access-control",
+		Short:   `Rule based Access Control for Databricks Resources.`,
+		Long:    `Rule based Access Control for Databricks Resources.`,
+		GroupID: "iam",
+		Annotations: map[string]string{
+			"package": "iam",
+		},
+
+		// This service is being previewed; hide from help output.
+		Hidden: true,
+	}
+
+	// Add methods
+	cmd.AddCommand(newCheckPolicy())
+
+	// Apply optional overrides to this command.
+	for _, fn := range cmdOverrides {
+		fn(cmd)
+	}
+
+	return cmd
+}
+
+// start check-policy command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var checkPolicyOverrides []func(
+	*cobra.Command,
+	*iam.CheckPolicyRequest,
+)
+
+func newCheckPolicy() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var checkPolicyReq iam.CheckPolicyRequest
+	var checkPolicyJson flags.JsonFlag
+
+	// TODO: short flags
+	cmd.Flags().Var(&checkPolicyJson, "json", `either inline JSON string or @path/to/file.json with request body`)
+
+	// TODO: complex arg: resource_info
+
+	cmd.Use = "check-policy"
+	cmd.Short = `Check access policy to a resource.`
+	cmd.Long = `Check access policy to a resource.`
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.PreRunE = root.MustWorkspaceClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		w := root.WorkspaceClient(ctx)
+
+		if cmd.Flags().Changed("json") {
+			diags := checkPolicyJson.Unmarshal(&checkPolicyReq)
+			if diags.HasError() {
+				return diags.Error()
+			}
+			if len(diags) > 0 {
+				err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags)
+				if err != nil {
+					return err
+				}
+			}
+		} else {
+			return fmt.Errorf("please provide command input in JSON format by specifying the --json flag")
+		}
+
+		response, err := w.AccessControl.CheckPolicy(ctx, checkPolicyReq)
+		if err != nil {
+			return err
+		}
+		return cmdio.Render(ctx, response)
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range checkPolicyOverrides {
+		fn(cmd, &checkPolicyReq)
+	}
+
+	return cmd
+}
+
+// end service AccessControl
diff --git a/cmd/workspace/cmd.go b/cmd/workspace/cmd.go
index f07d0cf76..c447bd736 100755
--- a/cmd/workspace/cmd.go
+++ b/cmd/workspace/cmd.go
@@ -3,6 +3,7 @@
 package workspace
 
 import (
+	access_control "github.com/databricks/cli/cmd/workspace/access-control"
 	alerts "github.com/databricks/cli/cmd/workspace/alerts"
 	alerts_legacy "github.com/databricks/cli/cmd/workspace/alerts-legacy"
 	apps "github.com/databricks/cli/cmd/workspace/apps"
@@ -96,6 +97,7 @@ import (
 func All() []*cobra.Command {
 	var out []*cobra.Command
 
+	out = append(out, access_control.New())
 	out = append(out, alerts.New())
 	out = append(out, alerts_legacy.New())
 	out = append(out, apps.New())
diff --git a/cmd/workspace/providers/providers.go b/cmd/workspace/providers/providers.go
index 504beac5e..4d6262cff 100755
--- a/cmd/workspace/providers/providers.go
+++ b/cmd/workspace/providers/providers.go
@@ -64,7 +64,7 @@ func newCreate() *cobra.Command {
 	cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
 	cmd.Flags().StringVar(&createReq.Comment, "comment", createReq.Comment, `Description about the provider.`)
-	cmd.Flags().StringVar(&createReq.RecipientProfileStr, "recipient-profile-str", createReq.RecipientProfileStr, `This field is required when the __authentication_type__ is **TOKEN** or not provided.`)
+	cmd.Flags().StringVar(&createReq.RecipientProfileStr, "recipient-profile-str", createReq.RecipientProfileStr, `This field is required when the __authentication_type__ is **TOKEN**, **OAUTH_CLIENT_CREDENTIALS** or not provided.`)
 
 	cmd.Use = "create NAME AUTHENTICATION_TYPE"
 	cmd.Short = `Create an auth provider.`
@@ -430,7 +430,7 @@ func newUpdate() *cobra.Command {
 	cmd.Flags().StringVar(&updateReq.Comment, "comment", updateReq.Comment, `Description about the provider.`)
 	cmd.Flags().StringVar(&updateReq.NewName, "new-name", updateReq.NewName, `New name for the provider.`)
 	cmd.Flags().StringVar(&updateReq.Owner, "owner", updateReq.Owner, `Username of Provider owner.`)
-	cmd.Flags().StringVar(&updateReq.RecipientProfileStr, "recipient-profile-str", updateReq.RecipientProfileStr, `This field is required when the __authentication_type__ is **TOKEN** or not provided.`)
+	cmd.Flags().StringVar(&updateReq.RecipientProfileStr, "recipient-profile-str", updateReq.RecipientProfileStr, `This field is required when the __authentication_type__ is **TOKEN**, **OAUTH_CLIENT_CREDENTIALS** or not provided.`)
 
 	cmd.Use = "update NAME"
 	cmd.Short = `Update a provider.`
diff --git a/cmd/workspace/recipients/recipients.go b/cmd/workspace/recipients/recipients.go
index 56abd2014..6d6ce42f1 100755
--- a/cmd/workspace/recipients/recipients.go
+++ b/cmd/workspace/recipients/recipients.go
@@ -91,7 +91,7 @@ func newCreate() *cobra.Command {
 	cmd.Long = `Create a share recipient.
   
   Creates a new recipient with the delta sharing authentication type in the
-  metastore. The caller must be a metastore admin or has the
+  metastore. The caller must be a metastore admin or have the
   **CREATE_RECIPIENT** privilege on the metastore.
 
   Arguments:
@@ -186,28 +186,16 @@ func newDelete() *cobra.Command {
 
 	cmd.Annotations = make(map[string]string)
 
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
 	cmd.PreRunE = root.MustWorkspaceClient
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
 
-		if len(args) == 0 {
-			promptSpinner := cmdio.Spinner(ctx)
-			promptSpinner <- "No NAME argument specified. Loading names for Recipients drop-down."
-			names, err := w.Recipients.RecipientInfoNameToMetastoreIdMap(ctx, sharing.ListRecipientsRequest{})
-			close(promptSpinner)
-			if err != nil {
-				return fmt.Errorf("failed to load names for Recipients drop-down. Please manually specify required arguments. Original error: %w", err)
-			}
-			id, err := cmdio.Select(ctx, names, "Name of the recipient")
-			if err != nil {
-				return err
-			}
-			args = append(args, id)
-		}
-		if len(args) != 1 {
-			return fmt.Errorf("expected to have name of the recipient")
-		}
 		deleteReq.Name = args[0]
 
 		err = w.Recipients.Delete(ctx, deleteReq)
@@ -258,28 +246,16 @@ func newGet() *cobra.Command {
 
 	cmd.Annotations = make(map[string]string)
 
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
 	cmd.PreRunE = root.MustWorkspaceClient
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
 
-		if len(args) == 0 {
-			promptSpinner := cmdio.Spinner(ctx)
-			promptSpinner <- "No NAME argument specified. Loading names for Recipients drop-down."
-			names, err := w.Recipients.RecipientInfoNameToMetastoreIdMap(ctx, sharing.ListRecipientsRequest{})
-			close(promptSpinner)
-			if err != nil {
-				return fmt.Errorf("failed to load names for Recipients drop-down. Please manually specify required arguments. Original error: %w", err)
-			}
-			id, err := cmdio.Select(ctx, names, "Name of the recipient")
-			if err != nil {
-				return err
-			}
-			args = append(args, id)
-		}
-		if len(args) != 1 {
-			return fmt.Errorf("expected to have name of the recipient")
-		}
 		getReq.Name = args[0]
 
 		response, err := w.Recipients.Get(ctx, getReq)
@@ -384,7 +360,7 @@ func newRotateToken() *cobra.Command {
   the provided token info. The caller must be the owner of the recipient.
 
   Arguments:
-    NAME: The name of the recipient.
+    NAME: The name of the Recipient.
     EXISTING_TOKEN_EXPIRE_IN_SECONDS: The expiration time of the bearer token in ISO 8601 format. This will set
       the expiration_time of existing token only to a smaller timestamp, it
       cannot extend the expiration_time. Use 0 to expire the existing token
@@ -479,28 +455,16 @@ func newSharePermissions() *cobra.Command {
 
 	cmd.Annotations = make(map[string]string)
 
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
 	cmd.PreRunE = root.MustWorkspaceClient
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
 
-		if len(args) == 0 {
-			promptSpinner := cmdio.Spinner(ctx)
-			promptSpinner <- "No NAME argument specified. Loading names for Recipients drop-down."
-			names, err := w.Recipients.RecipientInfoNameToMetastoreIdMap(ctx, sharing.ListRecipientsRequest{})
-			close(promptSpinner)
-			if err != nil {
-				return fmt.Errorf("failed to load names for Recipients drop-down. Please manually specify required arguments. Original error: %w", err)
-			}
-			id, err := cmdio.Select(ctx, names, "The name of the Recipient")
-			if err != nil {
-				return err
-			}
-			args = append(args, id)
-		}
-		if len(args) != 1 {
-			return fmt.Errorf("expected to have the name of the recipient")
-		}
 		sharePermissionsReq.Name = args[0]
 
 		response, err := w.Recipients.SharePermissions(ctx, sharePermissionsReq)
@@ -560,6 +524,11 @@ func newUpdate() *cobra.Command {
 
 	cmd.Annotations = make(map[string]string)
 
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
 	cmd.PreRunE = root.MustWorkspaceClient
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
@@ -577,30 +546,13 @@ func newUpdate() *cobra.Command {
 				}
 			}
 		}
-		if len(args) == 0 {
-			promptSpinner := cmdio.Spinner(ctx)
-			promptSpinner <- "No NAME argument specified. Loading names for Recipients drop-down."
-			names, err := w.Recipients.RecipientInfoNameToMetastoreIdMap(ctx, sharing.ListRecipientsRequest{})
-			close(promptSpinner)
-			if err != nil {
-				return fmt.Errorf("failed to load names for Recipients drop-down. Please manually specify required arguments. Original error: %w", err)
-			}
-			id, err := cmdio.Select(ctx, names, "Name of the recipient")
-			if err != nil {
-				return err
-			}
-			args = append(args, id)
-		}
-		if len(args) != 1 {
-			return fmt.Errorf("expected to have name of the recipient")
-		}
 		updateReq.Name = args[0]
 
-		err = w.Recipients.Update(ctx, updateReq)
+		response, err := w.Recipients.Update(ctx, updateReq)
 		if err != nil {
 			return err
 		}
-		return nil
+		return cmdio.Render(ctx, response)
 	}
 
 	// Disable completions since they are not applicable.
diff --git a/cmd/workspace/serving-endpoints/serving-endpoints.go b/cmd/workspace/serving-endpoints/serving-endpoints.go
index cc99177c7..034133623 100755
--- a/cmd/workspace/serving-endpoints/serving-endpoints.go
+++ b/cmd/workspace/serving-endpoints/serving-endpoints.go
@@ -49,6 +49,7 @@ func New() *cobra.Command {
 	cmd.AddCommand(newGetOpenApi())
 	cmd.AddCommand(newGetPermissionLevels())
 	cmd.AddCommand(newGetPermissions())
+	cmd.AddCommand(newHttpRequest())
 	cmd.AddCommand(newList())
 	cmd.AddCommand(newLogs())
 	cmd.AddCommand(newPatch())
@@ -153,16 +154,34 @@ func newCreate() *cobra.Command {
 	cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
 	// TODO: complex arg: ai_gateway
+	// TODO: complex arg: config
 	// TODO: array: rate_limits
 	cmd.Flags().BoolVar(&createReq.RouteOptimized, "route-optimized", createReq.RouteOptimized, `Enable route optimization for the serving endpoint.`)
 	// TODO: array: tags
 
-	cmd.Use = "create"
+	cmd.Use = "create NAME"
 	cmd.Short = `Create a new serving endpoint.`
-	cmd.Long = `Create a new serving endpoint.`
+	cmd.Long = `Create a new serving endpoint.
+
+  Arguments:
+    NAME: The name of the serving endpoint. This field is required and must be
+      unique across a Databricks workspace. An endpoint name can consist of
+      alphanumeric characters, dashes, and underscores.`
 
 	cmd.Annotations = make(map[string]string)
 
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		if cmd.Flags().Changed("json") {
+			err := root.ExactArgs(0)(cmd, args)
+			if err != nil {
+				return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'name' in your JSON input")
+			}
+			return nil
+		}
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
 	cmd.PreRunE = root.MustWorkspaceClient
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
@@ -179,8 +198,9 @@ func newCreate() *cobra.Command {
 					return err
 				}
 			}
-		} else {
-			return fmt.Errorf("please provide command input in JSON format by specifying the --json flag")
+		}
+		if !cmd.Flags().Changed("json") {
+			createReq.Name = args[0]
 		}
 
 		wait, err := w.ServingEndpoints.Create(ctx, createReq)
@@ -233,10 +253,7 @@ func newDelete() *cobra.Command {
 
 	cmd.Use = "delete NAME"
 	cmd.Short = `Delete a serving endpoint.`
-	cmd.Long = `Delete a serving endpoint.
-
-  Arguments:
-    NAME: The name of the serving endpoint. This field is required.`
+	cmd.Long = `Delete a serving endpoint.`
 
 	cmd.Annotations = make(map[string]string)
 
@@ -432,11 +449,12 @@ func newGetOpenApi() *cobra.Command {
 
 		getOpenApiReq.Name = args[0]
 
-		err = w.ServingEndpoints.GetOpenApi(ctx, getOpenApiReq)
+		response, err := w.ServingEndpoints.GetOpenApi(ctx, getOpenApiReq)
 		if err != nil {
 			return err
 		}
-		return nil
+		defer response.Contents.Close()
+		return cmdio.Render(ctx, response.Contents)
 	}
 
 	// Disable completions since they are not applicable.
@@ -568,6 +586,77 @@ func newGetPermissions() *cobra.Command {
 	return cmd
 }
 
+// start http-request command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var httpRequestOverrides []func(
+	*cobra.Command,
+	*serving.ExternalFunctionRequest,
+)
+
+func newHttpRequest() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var httpRequestReq serving.ExternalFunctionRequest
+
+	// TODO: short flags
+
+	cmd.Flags().StringVar(&httpRequestReq.Headers, "headers", httpRequestReq.Headers, `Additional headers for the request.`)
+	cmd.Flags().StringVar(&httpRequestReq.Json, "json", httpRequestReq.Json, `The JSON payload to send in the request body.`)
+	cmd.Flags().StringVar(&httpRequestReq.Params, "params", httpRequestReq.Params, `Query parameters for the request.`)
+
+	cmd.Use = "http-request CONNECTION_NAME METHOD PATH"
+	cmd.Short = `Make external services call using the credentials stored in UC Connection.`
+	cmd.Long = `Make external services call using the credentials stored in UC Connection.
+
+  Arguments:
+    CONNECTION_NAME: The connection name to use. This is required to identify the external
+      connection.
+    METHOD: The HTTP method to use (e.g., 'GET', 'POST').
+    PATH: The relative path for the API endpoint. This is required.`
+
+	// This command is being previewed; hide from help output.
+	cmd.Hidden = true
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(3)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustWorkspaceClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		w := root.WorkspaceClient(ctx)
+
+		httpRequestReq.ConnectionName = args[0]
+		_, err = fmt.Sscan(args[1], &httpRequestReq.Method)
+		if err != nil {
+			return fmt.Errorf("invalid METHOD: %s", args[1])
+		}
+		httpRequestReq.Path = args[2]
+
+		response, err := w.ServingEndpoints.HttpRequest(ctx, httpRequestReq)
+		if err != nil {
+			return err
+		}
+		return cmdio.Render(ctx, response)
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range httpRequestOverrides {
+		fn(cmd, &httpRequestReq)
+	}
+
+	return cmd
+}
+
 // start list command
 
 // Slice with functions to override default command behavior.
@@ -849,7 +938,7 @@ func newPutAiGateway() *cobra.Command {
 	cmd.Long = `Update AI Gateway of a serving endpoint.
   
   Used to update the AI Gateway of a serving endpoint. NOTE: Only external model
-  endpoints are currently supported.
+  and provisioned throughput endpoints are currently supported.
 
   Arguments:
     NAME: The name of the serving endpoint whose AI Gateway is being updated. This
diff --git a/go.mod b/go.mod
index 0ef800d7b..4a3bf1620 100644
--- a/go.mod
+++ b/go.mod
@@ -7,7 +7,7 @@ toolchain go1.23.4
 require (
 	github.com/Masterminds/semver/v3 v3.3.1 // MIT
 	github.com/briandowns/spinner v1.23.1 // Apache 2.0
-	github.com/databricks/databricks-sdk-go v0.55.0 // Apache 2.0
+	github.com/databricks/databricks-sdk-go v0.56.0 // Apache 2.0
 	github.com/fatih/color v1.18.0 // MIT
 	github.com/google/uuid v1.6.0 // BSD-3-Clause
 	github.com/hashicorp/go-version v1.7.0 // MPL 2.0
diff --git a/go.sum b/go.sum
index b1364cb26..b4e92c2c9 100644
--- a/go.sum
+++ b/go.sum
@@ -32,8 +32,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cyphar/filepath-securejoin v0.2.5 h1:6iR5tXJ/e6tJZzzdMc1km3Sa7RRIVBKAK32O2s7AYfo=
 github.com/cyphar/filepath-securejoin v0.2.5/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
-github.com/databricks/databricks-sdk-go v0.55.0 h1:ReziD6spzTDltM0ml80LggKo27F3oUjgTinCFDJDnak=
-github.com/databricks/databricks-sdk-go v0.55.0/go.mod h1:JpLizplEs+up9/Z4Xf2x++o3sM9eTTWFGzIXAptKJzI=
+github.com/databricks/databricks-sdk-go v0.56.0 h1:8BsqjrSLbm2ET+/SLCN8qD+v+HFvs891dzi1OaiyRfc=
+github.com/databricks/databricks-sdk-go v0.56.0/go.mod h1:JpLizplEs+up9/Z4Xf2x++o3sM9eTTWFGzIXAptKJzI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
diff --git a/integration/cmd/sync/sync_test.go b/integration/cmd/sync/sync_test.go
index 632497054..88e6ed89a 100644
--- a/integration/cmd/sync/sync_test.go
+++ b/integration/cmd/sync/sync_test.go
@@ -158,7 +158,7 @@ func (a *syncTest) remoteFileContent(ctx context.Context, relativePath, expected
 
 	var res []byte
 	a.c.Eventually(func() bool {
-		err = apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, &res)
+		err = apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, nil, &res)
 		require.NoError(a.t, err)
 		actualContent := string(res)
 		return actualContent == expectedContent
diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go
index 88bbadd32..7102b6e29 100644
--- a/libs/filer/files_client.go
+++ b/libs/filer/files_client.go
@@ -148,7 +148,7 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader,
 	overwrite := slices.Contains(mode, OverwriteIfExists)
 	urlPath = fmt.Sprintf("%s?overwrite=%t", urlPath, overwrite)
 	headers := map[string]string{"Content-Type": "application/octet-stream"}
-	err = w.apiClient.Do(ctx, http.MethodPut, urlPath, headers, reader, nil)
+	err = w.apiClient.Do(ctx, http.MethodPut, urlPath, headers, nil, reader, nil)
 
 	// Return early on success.
 	if err == nil {
@@ -176,7 +176,7 @@ func (w *FilesClient) Read(ctx context.Context, name string) (io.ReadCloser, err
 	}
 
 	var reader io.ReadCloser
-	err = w.apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, &reader)
+	err = w.apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, nil, &reader)
 
 	// Return early on success.
 	if err == nil {
diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go
index 8d5148edd..f7e1b3adb 100644
--- a/libs/filer/workspace_files_client.go
+++ b/libs/filer/workspace_files_client.go
@@ -106,7 +106,7 @@ func (info *wsfsFileInfo) MarshalJSON() ([]byte, error) {
 // as an interface to allow for mocking in tests.
 type apiClient interface {
 	Do(ctx context.Context, method, path string,
-		headers map[string]string, request, response any,
+		headers map[string]string, queryParams map[string]any, request, response any,
 		visitors ...func(*http.Request) error) error
 }
 
@@ -156,7 +156,7 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io
 		return err
 	}
 
-	err = w.apiClient.Do(ctx, http.MethodPost, urlPath, nil, body, nil)
+	err = w.apiClient.Do(ctx, http.MethodPost, urlPath, nil, nil, body, nil)
 
 	// Return early on success.
 	if err == nil {
@@ -341,6 +341,7 @@ func (w *WorkspaceFilesClient) Stat(ctx context.Context, name string) (fs.FileIn
 		http.MethodGet,
 		"/api/2.0/workspace/get-status",
 		nil,
+		nil,
 		map[string]string{
 			"path":               absPath,
 			"return_export_info": "true",
diff --git a/libs/filer/workspace_files_extensions_client_test.go b/libs/filer/workspace_files_extensions_client_test.go
index 9ea837fa9..f9c65d6ee 100644
--- a/libs/filer/workspace_files_extensions_client_test.go
+++ b/libs/filer/workspace_files_extensions_client_test.go
@@ -17,7 +17,7 @@ type mockApiClient struct {
 }
 
 func (m *mockApiClient) Do(ctx context.Context, method, path string,
-	headers map[string]string, request, response any,
+	headers map[string]string, queryParams map[string]any, request, response any,
 	visitors ...func(*http.Request) error,
 ) error {
 	args := m.Called(ctx, method, path, headers, request, response, visitors)
diff --git a/libs/git/info.go b/libs/git/info.go
index 46e57be48..dc4af9b6d 100644
--- a/libs/git/info.go
+++ b/libs/git/info.go
@@ -66,6 +66,7 @@ func fetchRepositoryInfoAPI(ctx context.Context, path string, w *databricks.Work
 		http.MethodGet,
 		apiEndpoint,
 		nil,
+		nil,
 		map[string]string{
 			"path":            path,
 			"return_git_info": "true",

From 1f63aa0912705f6722873c8d4d1389c398f4d2df Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 23 Jan 2025 12:46:22 +0100
Subject: [PATCH 134/247] tests: Improve reporting in case of FS errors (#2216)

## Changes
If there are unreadable files in a directory, raise an error but
continue with further diagnostics, because the answer is in the script
output.

## Tests
Manually - I'm working on some tests that create unreadable files, the
report is much better with this change.
---
 acceptance/acceptance_test.go | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 56db6ec20..96c1f651c 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -232,8 +232,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 	}
 
 	// Make sure there are not unaccounted for new files
-	files, err := ListDir(t, tmpDir)
-	require.NoError(t, err)
+	files := ListDir(t, tmpDir)
 	for _, relPath := range files {
 		if _, ok := inputs[relPath]; ok {
 			continue
@@ -450,11 +449,15 @@ func CopyDir(src, dst string, inputs, outputs map[string]bool) error {
 	})
 }
 
-func ListDir(t *testing.T, src string) ([]string, error) {
+func ListDir(t *testing.T, src string) []string {
 	var files []string
 	err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
-			return err
+			// Do not FailNow here.
+			// The output comparison is happening after this call which includes output.txt which
+			// includes errors printed by commands which include explanation why a given file cannot be read.
+			t.Errorf("Error when listing %s: path=%s: %s", src, path, err)
+			return nil
 		}
 
 		if info.IsDir() {
@@ -469,5 +472,8 @@ func ListDir(t *testing.T, src string) ([]string, error) {
 		files = append(files, relPath)
 		return nil
 	})
-	return files, err
+	if err != nil {
+		t.Errorf("Failed to list %s: %s", src, err)
+	}
+	return files
 }

From ddd45e25ee24cfad9ec5834ed66b71bb278b168d Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 23 Jan 2025 13:48:47 +0100
Subject: [PATCH 135/247] Pass USE_SDK_V2_{RESOURCES,DATA_SOURCES} to terraform
 (#2207)

## Changes
- Propagate env vars USE_SDK_V2_RESOURCES and $USE_SDK_V2_DATA_SOURCES
to terraform
- This are troubleshooting helpers for resources migrated to new plugin
framework, recommended here:
https://registry.terraform.io/providers/databricks/databricks/latest/docs/guides/troubleshooting#plugin-framework-migration-problems
- This current unblocks deploying quality monitors, see
https://github.com/databricks/terraform-provider-databricks/issues/4229#issuecomment-2520344690

## Tests
Manually testing that I can deploy quality monitor after this change
with `USE_SDK_V2_RESOURCES="databricks_quality_monitor"` set

### Main branch:
```
~/work/databricks_quality_monitor_repro % USE_SDK_V2_RESOURCES="databricks_quality_monitor" ../cli/cli-main bundle deploy
Uploading bundle files to /Workspace/Users/denis.bilenko@databricks.com/.bundle/quality_monitor_bundle/default/files...
Deploying resources...
Updating deployment state...
Deployment complete!
Error: terraform apply: exit status 1

Error: Provider produced inconsistent result after apply

When applying changes to databricks_quality_monitor.monitor_trips, provider
"provider[\"registry.terraform.io/databricks/databricks\"]" produced an
unexpected new value: .data_classification_config: block count changed from 0
to 1.

This is a bug in the provider, which should be reported in the provider's own
issue tracker.
```

### This branch:
```
~/work/databricks_quality_monitor_repro % USE_SDK_V2_RESOURCES="databricks_quality_monitor" ../cli/cli bundle deploy
Uploading bundle files to /Workspace/Users/denis.bilenko@databricks.com/.bundle/quality_monitor_bundle/default/files...
Deploying resources...
Updating deployment state...
Deployment complete!
```

### Config:
```
~/work/databricks_quality_monitor_repro % cat databricks.yml
bundle:
  name: quality_monitor_bundle

resources:
  quality_monitors:
    monitor_trips:
      table_name: main.denis-bilenko-cuj-pe34.trips_sanitized_1
      output_schema_name: main.denis-bilenko-cuj-pe34
      assets_dir: /Workspace/Users/${workspace.current_user.userName}/quality_monitor_issue
      snapshot: {}
```
---
 bundle/deploy/terraform/init.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/bundle/deploy/terraform/init.go b/bundle/deploy/terraform/init.go
index 6a014a7c1..5957611a4 100644
--- a/bundle/deploy/terraform/init.go
+++ b/bundle/deploy/terraform/init.go
@@ -108,6 +108,14 @@ var envCopy = []string{
 	// Include $TF_CLI_CONFIG_FILE to override terraform provider in development.
 	// See: https://developer.hashicorp.com/terraform/cli/config/config-file#explicit-installation-method-configuration
 	"TF_CLI_CONFIG_FILE",
+
+	// Include $USE_SDK_V2_RESOURCES and $USE_SDK_V2_DATA_SOURCES, these are used to switch back from plugin framework to SDKv2.
+	// This is used for mitigation issues with resource migrated to plugin framework, as recommended here:
+	// https://registry.terraform.io/providers/databricks/databricks/latest/docs/guides/troubleshooting#plugin-framework-migration-problems
+	// It is currently a workaround for deploying quality_monitors
+	// https://github.com/databricks/terraform-provider-databricks/issues/4229#issuecomment-2520344690
+	"USE_SDK_V2_RESOURCES",
+	"USE_SDK_V2_DATA_SOURCES",
 }
 
 // This function inherits some environment variables for Terraform CLI.

From 6153423c56ff56583087e0fa1c92a02a2eb3dca2 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Thu, 23 Jan 2025 14:21:59 +0100
Subject: [PATCH 136/247] Revert "Upgrade Go SDK to 0.56.0 (#2214)" (#2217)

This reverts commit 798189eb96bc1184119dc039a2728f87b4ce6212.
---
 .codegen/_openapi_sha                         |   2 +-
 .codegen/service.go.tmpl                      |  20 +-
 .gitattributes                                |   1 -
 bundle/deploy/terraform/convert_test.go       |   4 +-
 .../convert_model_serving_endpoint_test.go    |   2 +-
 .../internal/schema/annotations_openapi.yml   | 367 +++++++-----------
 .../schema/annotations_openapi_overrides.yml  |  11 -
 bundle/schema/jsonschema.json                 | 173 ++++-----
 .../custom-app-integration.go                 |   1 -
 cmd/api/api.go                                |   2 +-
 .../access-control/access-control.go          | 109 ------
 cmd/workspace/cmd.go                          |   2 -
 cmd/workspace/providers/providers.go          |   4 +-
 cmd/workspace/recipients/recipients.go        |  96 +++--
 .../serving-endpoints/serving-endpoints.go    | 111 +-----
 go.mod                                        |   2 +-
 go.sum                                        |   4 +-
 integration/cmd/sync/sync_test.go             |   2 +-
 libs/filer/files_client.go                    |   4 +-
 libs/filer/workspace_files_client.go          |   5 +-
 .../workspace_files_extensions_client_test.go |   2 +-
 libs/git/info.go                              |   1 -
 22 files changed, 338 insertions(+), 587 deletions(-)
 delete mode 100755 cmd/workspace/access-control/access-control.go

diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha
index 588cf9d63..dfe78790a 100644
--- a/.codegen/_openapi_sha
+++ b/.codegen/_openapi_sha
@@ -1 +1 @@
-0be1b914249781b5e903b7676fd02255755bc851
\ No newline at end of file
+779817ed8d63031f5ea761fbd25ee84f38feec0d
\ No newline at end of file
diff --git a/.codegen/service.go.tmpl b/.codegen/service.go.tmpl
index 2f4987b13..0c9fa089a 100644
--- a/.codegen/service.go.tmpl
+++ b/.codegen/service.go.tmpl
@@ -109,19 +109,16 @@ var {{.CamelName}}Overrides []func(
 	{{- end }}
 )
 
-{{- $excludeFromJson := list "http-request"}}
-
 func new{{.PascalName}}() *cobra.Command {
 	cmd := &cobra.Command{}
 
-	{{- $canUseJson := and .CanUseJson (not (in $excludeFromJson .KebabName )) -}}
 	{{- if .Request}}
 
 	var {{.CamelName}}Req {{.Service.Package.Name}}.{{.Request.PascalName}}
 	{{- if .RequestBodyField }}
 	{{.CamelName}}Req.{{.RequestBodyField.PascalName}} = &{{.Service.Package.Name}}.{{.RequestBodyField.Entity.PascalName}}{}
 	{{- end }}
-	{{- if $canUseJson}}
+	{{- if .CanUseJson}}
 	var {{.CamelName}}Json flags.JsonFlag
 	{{- end}}
 	{{- end}}
@@ -138,7 +135,7 @@ func new{{.PascalName}}() *cobra.Command {
 	{{- $request = .RequestBodyField.Entity -}}
 	{{- end -}}
     {{if $request }}// TODO: short flags
-	{{- if $canUseJson}}
+	{{- if .CanUseJson}}
 	cmd.Flags().Var(&{{.CamelName}}Json, "json", `either inline JSON string or @path/to/file.json with request body`)
 	{{- end}}
     {{$method := .}}
@@ -180,7 +177,7 @@ func new{{.PascalName}}() *cobra.Command {
 	{{- $hasRequiredArgs :=  and (not $hasIdPrompt) $hasPosArgs -}}
 	{{- $hasSingleRequiredRequestBodyFieldWithPrompt := and (and $hasIdPrompt $request) (eq 1 (len $request.RequiredRequestBodyFields))  -}}
 	{{- $onlyPathArgsRequiredAsPositionalArguments := and $request (eq (len .RequiredPositionalArguments) (len $request.RequiredPathFields)) -}}
-	{{- $hasDifferentArgsWithJsonFlag := and (not $onlyPathArgsRequiredAsPositionalArguments) (and $canUseJson (or $request.HasRequiredRequestBodyFields )) -}}
+	{{- $hasDifferentArgsWithJsonFlag := and (not $onlyPathArgsRequiredAsPositionalArguments) (and .CanUseJson (or $request.HasRequiredRequestBodyFields )) -}}
 	{{- $hasCustomArgHandler := or $hasRequiredArgs $hasDifferentArgsWithJsonFlag -}}
 
 	{{- $atleastOneArgumentWithDescription := false -}}
@@ -242,7 +239,7 @@ func new{{.PascalName}}() *cobra.Command {
 		ctx := cmd.Context()
 		{{if .Service.IsAccounts}}a := root.AccountClient(ctx){{else}}w := root.WorkspaceClient(ctx){{end}}
 		{{- if .Request }}
-			{{ if $canUseJson }}
+			{{ if .CanUseJson }}
 			if cmd.Flags().Changed("json") {
 					diags := {{.CamelName}}Json.Unmarshal(&{{.CamelName}}Req{{ if .RequestBodyField }}.{{.RequestBodyField.PascalName}}{{ end }})
 					if diags.HasError() {
@@ -258,7 +255,7 @@ func new{{.PascalName}}() *cobra.Command {
 				return fmt.Errorf("please provide command input in JSON format by specifying the --json flag")
 			}{{- end}}
 			{{- if $hasPosArgs }}
-			{{- if and $canUseJson $hasSingleRequiredRequestBodyFieldWithPrompt }} else {
+			{{- if and .CanUseJson $hasSingleRequiredRequestBodyFieldWithPrompt }} else {
 			{{- end}}
 			{{- if $hasIdPrompt}}
 				if len(args) == 0 {
@@ -282,9 +279,9 @@ func new{{.PascalName}}() *cobra.Command {
 
 			{{$method := .}}
 			{{- range $arg, $field := .RequiredPositionalArguments}}
-				{{- template "args-scan" (dict "Arg" $arg "Field" $field "Method" $method "HasIdPrompt" $hasIdPrompt "ExcludeFromJson" $excludeFromJson)}}
+				{{- template "args-scan" (dict "Arg" $arg "Field" $field "Method" $method "HasIdPrompt" $hasIdPrompt)}}
 			{{- end -}}
-			{{- if and $canUseJson $hasSingleRequiredRequestBodyFieldWithPrompt }}
+			{{- if and .CanUseJson $hasSingleRequiredRequestBodyFieldWithPrompt }}
 			}
 			{{- end}}
 
@@ -395,8 +392,7 @@ func new{{.PascalName}}() *cobra.Command {
 	{{- $method := .Method -}}
 	{{- $arg := .Arg -}}
 	{{- $hasIdPrompt := .HasIdPrompt -}}
-	{{ $canUseJson := and $method.CanUseJson (not (in .ExcludeFromJson $method.KebabName)) }}
-	{{- $optionalIfJsonIsUsed := and (not $hasIdPrompt) (and $field.IsRequestBodyField $canUseJson) }}
+	{{- $optionalIfJsonIsUsed := and (not $hasIdPrompt) (and $field.IsRequestBodyField $method.CanUseJson) }}
 	{{- if $optionalIfJsonIsUsed  }}
 	if !cmd.Flags().Changed("json") {
 	{{- end }}
diff --git a/.gitattributes b/.gitattributes
index ebe94ed8e..0a8ddf3cb 100755
--- a/.gitattributes
+++ b/.gitattributes
@@ -31,7 +31,6 @@ cmd/account/users/users.go linguist-generated=true
 cmd/account/vpc-endpoints/vpc-endpoints.go linguist-generated=true
 cmd/account/workspace-assignment/workspace-assignment.go linguist-generated=true
 cmd/account/workspaces/workspaces.go linguist-generated=true
-cmd/workspace/access-control/access-control.go linguist-generated=true
 cmd/workspace/aibi-dashboard-embedding-access-policy/aibi-dashboard-embedding-access-policy.go linguist-generated=true
 cmd/workspace/aibi-dashboard-embedding-approved-domains/aibi-dashboard-embedding-approved-domains.go linguist-generated=true
 cmd/workspace/alerts-legacy/alerts-legacy.go linguist-generated=true
diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go
index afc1fb22a..ffe55db71 100644
--- a/bundle/deploy/terraform/convert_test.go
+++ b/bundle/deploy/terraform/convert_test.go
@@ -419,7 +419,7 @@ func TestBundleToTerraformModelServing(t *testing.T) {
 	src := resources.ModelServingEndpoint{
 		CreateServingEndpoint: &serving.CreateServingEndpoint{
 			Name: "name",
-			Config: &serving.EndpointCoreConfigInput{
+			Config: serving.EndpointCoreConfigInput{
 				ServedModels: []serving.ServedModelInput{
 					{
 						ModelName:          "model_name",
@@ -474,7 +474,7 @@ func TestBundleToTerraformModelServingPermissions(t *testing.T) {
 			// and as such observed the `omitempty` tag.
 			// The new method leverages [dyn.Value] where any field that is not
 			// explicitly set is not part of the value.
-			Config: &serving.EndpointCoreConfigInput{
+			Config: serving.EndpointCoreConfigInput{
 				ServedModels: []serving.ServedModelInput{
 					{
 						ModelName:          "model_name",
diff --git a/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go b/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go
index 98cf2dc22..d46350bb7 100644
--- a/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go
+++ b/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go
@@ -17,7 +17,7 @@ func TestConvertModelServingEndpoint(t *testing.T) {
 	src := resources.ModelServingEndpoint{
 		CreateServingEndpoint: &serving.CreateServingEndpoint{
 			Name: "name",
-			Config: &serving.EndpointCoreConfigInput{
+			Config: serving.EndpointCoreConfigInput{
 				ServedModels: []serving.ServedModelInput{
 					{
 						ModelName:          "model_name",
diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml
index d5a9bf69e..8ff5c9253 100644
--- a/bundle/internal/schema/annotations_openapi.yml
+++ b/bundle/internal/schema/annotations_openapi.yml
@@ -353,12 +353,12 @@ github.com/databricks/cli/bundle/config/resources.MlflowModel:
 github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint:
   "ai_gateway":
     "description": |-
-      The AI Gateway configuration for the serving endpoint. NOTE: Only external model and provisioned throughput endpoints are currently supported.
+      The AI Gateway configuration for the serving endpoint. NOTE: only external model endpoints are supported as of now.
   "config":
     "description": |-
       The core config of the serving endpoint.
   "name":
-    "description": |-
+    "description": |
       The name of the serving endpoint. This field is required and must be unique across a Databricks workspace.
       An endpoint name can consist of alphanumeric characters, dashes, and underscores.
   "rate_limits":
@@ -1974,9 +1974,6 @@ github.com/databricks/databricks-sdk-go/service/jobs.SparkJarTask:
       Parameters passed to the main method.
 
       Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.
-  "run_as_repl":
-    "description": |-
-      Deprecated. A value of `false` is no longer supported.
 github.com/databricks/databricks-sdk-go/service/jobs.SparkPythonTask:
   "parameters":
     "description": |-
@@ -2687,36 +2684,27 @@ github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfigScd
 github.com/databricks/databricks-sdk-go/service/serving.Ai21LabsConfig:
   "ai21labs_api_key":
     "description": |-
-      The Databricks secret key reference for an AI21 Labs API key. If you
-      prefer to paste your API key directly, see `ai21labs_api_key_plaintext`.
-      You must provide an API key using one of the following fields:
-      `ai21labs_api_key` or `ai21labs_api_key_plaintext`.
+      The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`.
   "ai21labs_api_key_plaintext":
     "description": |-
-      An AI21 Labs API key provided as a plaintext string. If you prefer to
-      reference your key using Databricks Secrets, see `ai21labs_api_key`. You
-      must provide an API key using one of the following fields:
-      `ai21labs_api_key` or `ai21labs_api_key_plaintext`.
+      An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`.
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayConfig:
   "guardrails":
     "description": |-
       Configuration for AI Guardrails to prevent unwanted data and unsafe data in requests and responses.
   "inference_table_config":
     "description": |-
-      Configuration for payload logging using inference tables.
-      Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.
+      Configuration for payload logging using inference tables. Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.
   "rate_limits":
     "description": |-
       Configuration for rate limits which can be set to limit endpoint traffic.
   "usage_tracking_config":
     "description": |-
-      Configuration to enable usage tracking using system tables.
-      These tables allow you to monitor operational usage on endpoints and their associated costs.
+      Configuration to enable usage tracking using system tables. These tables allow you to monitor operational usage on endpoints and their associated costs.
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParameters:
   "invalid_keywords":
     "description": |-
-      List of invalid keywords.
-      AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.
+      List of invalid keywords. AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.
   "pii":
     "description": |-
       Configuration for guardrail PII filter.
@@ -2725,14 +2713,15 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParame
       Indicates whether the safety filter is enabled.
   "valid_topics":
     "description": |-
-      The list of allowed topics.
-      Given a chat request, this guardrail flags the request if its topic is not in the allowed topics.
+      The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics.
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehavior:
   "behavior":
     "description": |-
-      Configuration for input guardrail filters.
+      Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehaviorBehavior:
   "_":
+    "description": |-
+      Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.
     "enum":
       - |-
         NONE
@@ -2748,32 +2737,30 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrails:
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayInferenceTableConfig:
   "catalog_name":
     "description": |-
-      The name of the catalog in Unity Catalog. Required when enabling inference tables.
-      NOTE: On update, you have to disable inference table first in order to change the catalog name.
+      The name of the catalog in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the catalog name.
   "enabled":
     "description": |-
       Indicates whether the inference table is enabled.
   "schema_name":
     "description": |-
-      The name of the schema in Unity Catalog. Required when enabling inference tables.
-      NOTE: On update, you have to disable inference table first in order to change the schema name.
+      The name of the schema in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the schema name.
   "table_name_prefix":
     "description": |-
-      The prefix of the table in Unity Catalog.
-      NOTE: On update, you have to disable inference table first in order to change the prefix name.
+      The prefix of the table in Unity Catalog. NOTE: On update, you have to disable inference table first in order to change the prefix name.
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit:
   "calls":
     "description": |-
       Used to specify how many calls are allowed for a key within the renewal_period.
   "key":
     "description": |-
-      Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported,
-      with 'endpoint' being the default if not specified.
+      Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.
   "renewal_period":
     "description": |-
       Renewal period field for a rate limit. Currently, only 'minute' is supported.
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitKey:
   "_":
+    "description": |-
+      Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.
     "enum":
       - |-
         user
@@ -2781,6 +2768,8 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitKey:
         endpoint
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitRenewalPeriod:
   "_":
+    "description": |-
+      Renewal period field for a rate limit. Currently, only 'minute' is supported.
     "enum":
       - |-
         minute
@@ -2791,43 +2780,26 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayUsageTrackingCo
 github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfig:
   "aws_access_key_id":
     "description": |-
-      The Databricks secret key reference for an AWS access key ID with
-      permissions to interact with Bedrock services. If you prefer to paste
-      your API key directly, see `aws_access_key_id_plaintext`. You must provide an API
-      key using one of the following fields: `aws_access_key_id` or
-      `aws_access_key_id_plaintext`.
+      The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.
   "aws_access_key_id_plaintext":
     "description": |-
-      An AWS access key ID with permissions to interact with Bedrock services
-      provided as a plaintext string. If you prefer to reference your key using
-      Databricks Secrets, see `aws_access_key_id`. You must provide an API key
-      using one of the following fields: `aws_access_key_id` or
-      `aws_access_key_id_plaintext`.
+      An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.
   "aws_region":
     "description": |-
       The AWS region to use. Bedrock has to be enabled there.
   "aws_secret_access_key":
     "description": |-
-      The Databricks secret key reference for an AWS secret access key paired
-      with the access key ID, with permissions to interact with Bedrock
-      services. If you prefer to paste your API key directly, see
-      `aws_secret_access_key_plaintext`. You must provide an API key using one
-      of the following fields: `aws_secret_access_key` or
-      `aws_secret_access_key_plaintext`.
+      The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.
   "aws_secret_access_key_plaintext":
     "description": |-
-      An AWS secret access key paired with the access key ID, with permissions
-      to interact with Bedrock services provided as a plaintext string. If you
-      prefer to reference your key using Databricks Secrets, see
-      `aws_secret_access_key`. You must provide an API key using one of the
-      following fields: `aws_secret_access_key` or
-      `aws_secret_access_key_plaintext`.
+      An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.
   "bedrock_provider":
     "description": |-
-      The underlying provider in Amazon Bedrock. Supported values (case
-      insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.
+      The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.
 github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfigBedrockProvider:
   "_":
+    "description": |-
+      The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.
     "enum":
       - |-
         anthropic
@@ -2840,16 +2812,10 @@ github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfigBedro
 github.com/databricks/databricks-sdk-go/service/serving.AnthropicConfig:
   "anthropic_api_key":
     "description": |-
-      The Databricks secret key reference for an Anthropic API key. If you
-      prefer to paste your API key directly, see `anthropic_api_key_plaintext`.
-      You must provide an API key using one of the following fields:
-      `anthropic_api_key` or `anthropic_api_key_plaintext`.
+      The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.
   "anthropic_api_key_plaintext":
     "description": |-
-      The Anthropic API key provided as a plaintext string. If you prefer to
-      reference your key using Databricks Secrets, see `anthropic_api_key`. You
-      must provide an API key using one of the following fields:
-      `anthropic_api_key` or `anthropic_api_key_plaintext`.
+      The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.
 github.com/databricks/databricks-sdk-go/service/serving.AutoCaptureConfigInput:
   "catalog_name":
     "description": |-
@@ -2865,58 +2831,42 @@ github.com/databricks/databricks-sdk-go/service/serving.AutoCaptureConfigInput:
       The prefix of the table in Unity Catalog. NOTE: On update, you cannot change the prefix name if the inference table is already enabled.
 github.com/databricks/databricks-sdk-go/service/serving.CohereConfig:
   "cohere_api_base":
-    "description": |-
-      This is an optional field to provide a customized base URL for the Cohere
-      API. If left unspecified, the standard Cohere base URL is used.
+    "description": "This is an optional field to provide a customized base URL for the Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n"
   "cohere_api_key":
     "description": |-
-      The Databricks secret key reference for a Cohere API key. If you prefer
-      to paste your API key directly, see `cohere_api_key_plaintext`. You must
-      provide an API key using one of the following fields: `cohere_api_key` or
-      `cohere_api_key_plaintext`.
+      The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.
   "cohere_api_key_plaintext":
     "description": |-
-      The Cohere API key provided as a plaintext string. If you prefer to
-      reference your key using Databricks Secrets, see `cohere_api_key`. You
-      must provide an API key using one of the following fields:
-      `cohere_api_key` or `cohere_api_key_plaintext`.
+      The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.
 github.com/databricks/databricks-sdk-go/service/serving.DatabricksModelServingConfig:
   "databricks_api_token":
-    "description": |-
-      The Databricks secret key reference for a Databricks API token that
-      corresponds to a user or service principal with Can Query access to the
-      model serving endpoint pointed to by this external model. If you prefer
-      to paste your API key directly, see `databricks_api_token_plaintext`. You
-      must provide an API key using one of the following fields:
-      `databricks_api_token` or `databricks_api_token_plaintext`.
+    "description": |
+      The Databricks secret key reference for a Databricks API token that corresponds to a user or service
+      principal with Can Query access to the model serving endpoint pointed to by this external model.
+      If you prefer to paste your API key directly, see `databricks_api_token_plaintext`.
+      You must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.
   "databricks_api_token_plaintext":
-    "description": |-
-      The Databricks API token that corresponds to a user or service principal
-      with Can Query access to the model serving endpoint pointed to by this
-      external model provided as a plaintext string. If you prefer to reference
-      your key using Databricks Secrets, see `databricks_api_token`. You must
-      provide an API key using one of the following fields:
-      `databricks_api_token` or `databricks_api_token_plaintext`.
+    "description": |
+      The Databricks API token that corresponds to a user or service
+      principal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string.
+      If you prefer to reference your key using Databricks Secrets, see `databricks_api_token`.
+      You must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.
   "databricks_workspace_url":
-    "description": |-
-      The URL of the Databricks workspace containing the model serving endpoint
-      pointed to by this external model.
+    "description": |
+      The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model.
 github.com/databricks/databricks-sdk-go/service/serving.EndpointCoreConfigInput:
   "auto_capture_config":
     "description": |-
       Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog.
-      Note: this field is deprecated for creating new provisioned throughput endpoints,
-      or updating existing provisioned throughput endpoints that never have inference table configured;
-      in these cases please use AI Gateway to manage inference tables.
   "served_entities":
     "description": |-
-      The list of served entities under the serving endpoint config.
+      A list of served entities for the endpoint to serve. A serving endpoint can have up to 15 served entities.
   "served_models":
     "description": |-
-      (Deprecated, use served_entities instead) The list of served models under the serving endpoint config.
+      (Deprecated, use served_entities instead) A list of served models for the endpoint to serve. A serving endpoint can have up to 15 served models.
   "traffic_config":
     "description": |-
-      The traffic configuration associated with the serving endpoint config.
+      The traffic config defining how invocations to the serving endpoint should be routed.
 github.com/databricks/databricks-sdk-go/service/serving.EndpointTag:
   "key":
     "description": |-
@@ -2953,13 +2903,17 @@ github.com/databricks/databricks-sdk-go/service/serving.ExternalModel:
     "description": |-
       PaLM Config. Only required if the provider is 'palm'.
   "provider":
-    "description": |-
-      The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic', 'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.
+    "description": |
+      The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',
+      'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.",
   "task":
     "description": |-
       The task type of the external model.
 github.com/databricks/databricks-sdk-go/service/serving.ExternalModelProvider:
   "_":
+    "description": |
+      The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',
+      'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.",
     "enum":
       - |-
         ai21labs
@@ -2980,114 +2934,70 @@ github.com/databricks/databricks-sdk-go/service/serving.ExternalModelProvider:
 github.com/databricks/databricks-sdk-go/service/serving.GoogleCloudVertexAiConfig:
   "private_key":
     "description": |-
-      The Databricks secret key reference for a private key for the service
-      account which has access to the Google Cloud Vertex AI Service. See [Best
-      practices for managing service account keys]. If you prefer to paste your
-      API key directly, see `private_key_plaintext`. You must provide an API
-      key using one of the following fields: `private_key` or
-      `private_key_plaintext`
-
-      [Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys
+      The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`
   "private_key_plaintext":
     "description": |-
-      The private key for the service account which has access to the Google
-      Cloud Vertex AI Service provided as a plaintext secret. See [Best
-      practices for managing service account keys]. If you prefer to reference
-      your key using Databricks Secrets, see `private_key`. You must provide an
-      API key using one of the following fields: `private_key` or
-      `private_key_plaintext`.
-
-      [Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys
+      The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`.
   "project_id":
     "description": |-
-      This is the Google Cloud project id that the service account is
-      associated with.
+      This is the Google Cloud project id that the service account is associated with.
   "region":
     "description": |-
-      This is the region for the Google Cloud Vertex AI Service. See [supported
-      regions] for more details. Some models are only available in specific
-      regions.
-
-      [supported regions]: https://cloud.google.com/vertex-ai/docs/general/locations
+      This is the region for the Google Cloud Vertex AI Service. See [supported regions](https://cloud.google.com/vertex-ai/docs/general/locations) for more details. Some models are only available in specific regions.
 github.com/databricks/databricks-sdk-go/service/serving.OpenAiConfig:
-  "_":
-    "description": |-
-      Configs needed to create an OpenAI model route.
   "microsoft_entra_client_id":
-    "description": |-
-      This field is only required for Azure AD OpenAI and is the Microsoft
-      Entra Client ID.
+    "description": |
+      This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.
   "microsoft_entra_client_secret":
-    "description": |-
-      The Databricks secret key reference for a client secret used for
-      Microsoft Entra ID authentication. If you prefer to paste your client
-      secret directly, see `microsoft_entra_client_secret_plaintext`. You must
-      provide an API key using one of the following fields:
-      `microsoft_entra_client_secret` or
-      `microsoft_entra_client_secret_plaintext`.
+    "description": |
+      The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication.
+      If you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`.
+      You must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.
   "microsoft_entra_client_secret_plaintext":
-    "description": |-
-      The client secret used for Microsoft Entra ID authentication provided as
-      a plaintext string. If you prefer to reference your key using Databricks
-      Secrets, see `microsoft_entra_client_secret`. You must provide an API key
-      using one of the following fields: `microsoft_entra_client_secret` or
-      `microsoft_entra_client_secret_plaintext`.
+    "description": |
+      The client secret used for Microsoft Entra ID authentication provided as a plaintext string.
+      If you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`.
+      You must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.
   "microsoft_entra_tenant_id":
-    "description": |-
-      This field is only required for Azure AD OpenAI and is the Microsoft
-      Entra Tenant ID.
+    "description": |
+      This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.
   "openai_api_base":
-    "description": |-
-      This is a field to provide a customized base URl for the OpenAI API. For
-      Azure OpenAI, this field is required, and is the base URL for the Azure
-      OpenAI API service provided by Azure. For other OpenAI API types, this
-      field is optional, and if left unspecified, the standard OpenAI base URL
-      is used.
+    "description": |
+      This is a field to provide a customized base URl for the OpenAI API.
+      For Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service
+      provided by Azure.
+      For other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used.
   "openai_api_key":
     "description": |-
-      The Databricks secret key reference for an OpenAI API key using the
-      OpenAI or Azure service. If you prefer to paste your API key directly,
-      see `openai_api_key_plaintext`. You must provide an API key using one of
-      the following fields: `openai_api_key` or `openai_api_key_plaintext`.
+      The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`.
   "openai_api_key_plaintext":
     "description": |-
-      The OpenAI API key using the OpenAI or Azure service provided as a
-      plaintext string. If you prefer to reference your key using Databricks
-      Secrets, see `openai_api_key`. You must provide an API key using one of
-      the following fields: `openai_api_key` or `openai_api_key_plaintext`.
+      The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`.
   "openai_api_type":
-    "description": |-
-      This is an optional field to specify the type of OpenAI API to use. For
-      Azure OpenAI, this field is required, and adjust this parameter to
-      represent the preferred security access validation protocol. For access
-      token validation, use azure. For authentication using Azure Active
+    "description": |
+      This is an optional field to specify the type of OpenAI API to use.
+      For Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security
+      access validation protocol. For access token validation, use azure. For authentication using Azure Active
       Directory (Azure AD) use, azuread.
   "openai_api_version":
-    "description": |-
-      This is an optional field to specify the OpenAI API version. For Azure
-      OpenAI, this field is required, and is the version of the Azure OpenAI
-      service to utilize, specified by a date.
+    "description": |
+      This is an optional field to specify the OpenAI API version.
+      For Azure OpenAI, this field is required, and is the version of the Azure OpenAI service to
+      utilize, specified by a date.
   "openai_deployment_name":
-    "description": |-
-      This field is only required for Azure OpenAI and is the name of the
-      deployment resource for the Azure OpenAI service.
+    "description": |
+      This field is only required for Azure OpenAI and is the name of the deployment resource for the
+      Azure OpenAI service.
   "openai_organization":
-    "description": |-
-      This is an optional field to specify the organization in OpenAI or Azure
-      OpenAI.
+    "description": |
+      This is an optional field to specify the organization in OpenAI or Azure OpenAI.
 github.com/databricks/databricks-sdk-go/service/serving.PaLmConfig:
   "palm_api_key":
     "description": |-
-      The Databricks secret key reference for a PaLM API key. If you prefer to
-      paste your API key directly, see `palm_api_key_plaintext`. You must
-      provide an API key using one of the following fields: `palm_api_key` or
-      `palm_api_key_plaintext`.
+      The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`.
   "palm_api_key_plaintext":
     "description": |-
-      The PaLM API key provided as a plaintext string. If you prefer to
-      reference your key using Databricks Secrets, see `palm_api_key`. You must
-      provide an API key using one of the following fields: `palm_api_key` or
-      `palm_api_key_plaintext`.
+      The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`.
 github.com/databricks/databricks-sdk-go/service/serving.RateLimit:
   "calls":
     "description": |-
@@ -3100,6 +3010,8 @@ github.com/databricks/databricks-sdk-go/service/serving.RateLimit:
       Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported.
 github.com/databricks/databricks-sdk-go/service/serving.RateLimitKey:
   "_":
+    "description": |-
+      Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.
     "enum":
       - |-
         user
@@ -3107,6 +3019,8 @@ github.com/databricks/databricks-sdk-go/service/serving.RateLimitKey:
         endpoint
 github.com/databricks/databricks-sdk-go/service/serving.RateLimitRenewalPeriod:
   "_":
+    "description": |-
+      Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported.
     "enum":
       - |-
         minute
@@ -3119,15 +3033,21 @@ github.com/databricks/databricks-sdk-go/service/serving.Route:
       The percentage of endpoint traffic to send to this route. It must be an integer between 0 and 100 inclusive.
 github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput:
   "entity_name":
+    "description": |
+      The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC),
+      or a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of
+      __catalog_name__.__schema_name__.__model_name__.
+  "entity_version":
     "description": |-
-      The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC), or a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of **catalog_name.schema_name.model_name**.
-  "entity_version": {}
+      The version of the model in Databricks Model Registry to be served or empty if the entity is a FEATURE_SPEC.
   "environment_vars":
-    "description": |-
-      An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{"OPENAI_API_KEY": "{{secrets/my_scope/my_key}}", "DATABRICKS_TOKEN": "{{secrets/my_scope2/my_key2}}"}`
+    "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity.\nNote: this is an experimental feature and subject to change. \nExample entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`"
   "external_model":
-    "description": |-
-      The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same.
+    "description": |
+      The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)
+      can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model,
+      it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later.
+      The task type of all external models within an endpoint must be the same.
   "instance_profile_arn":
     "description": |-
       ARN of the instance profile that the served entity uses to access AWS resources.
@@ -3138,46 +3058,68 @@ github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput:
     "description": |-
       The minimum tokens per second that the endpoint can scale down to.
   "name":
-    "description": |-
-      The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.
+    "description": |
+      The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores.
+      If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other
+      entities, it defaults to <entity-name>-<entity-version>.
   "scale_to_zero_enabled":
     "description": |-
       Whether the compute resources for the served entity should scale down to zero.
   "workload_size":
-    "description": |-
-      The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.
+    "description": |
+      The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between.
+      A single unit of provisioned concurrency can process one request at a time.
+      Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency).
+      If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.
   "workload_type":
-    "description": |-
-      The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).
+    "description": |
+      The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is
+      "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.
+      See the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).
 github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput:
   "environment_vars":
-    "description": |-
-      An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{"OPENAI_API_KEY": "{{secrets/my_scope/my_key}}", "DATABRICKS_TOKEN": "{{secrets/my_scope2/my_key2}}"}`
+    "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this model.\nNote: this is an experimental feature and subject to change. \nExample model environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`"
   "instance_profile_arn":
     "description": |-
-      ARN of the instance profile that the served entity uses to access AWS resources.
+      ARN of the instance profile that the served model will use to access AWS resources.
   "max_provisioned_throughput":
     "description": |-
       The maximum tokens per second that the endpoint can scale up to.
   "min_provisioned_throughput":
     "description": |-
       The minimum tokens per second that the endpoint can scale down to.
-  "model_name": {}
-  "model_version": {}
-  "name":
+  "model_name":
+    "description": |
+      The name of the model in Databricks Model Registry to be served or if the model resides in Unity Catalog, the full name of model,
+      in the form of __catalog_name__.__schema_name__.__model_name__.
+  "model_version":
     "description": |-
-      The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.
+      The version of the model in Databricks Model Registry or Unity Catalog to be served.
+  "name":
+    "description": |
+      The name of a served model. It must be unique across an endpoint. If not specified, this field will default to <model-name>-<model-version>.
+      A served model name can consist of alphanumeric characters, dashes, and underscores.
   "scale_to_zero_enabled":
     "description": |-
-      Whether the compute resources for the served entity should scale down to zero.
+      Whether the compute resources for the served model should scale down to zero.
   "workload_size":
-    "description": |-
-      The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.
+    "description": |
+      The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.
+      A single unit of provisioned concurrency can process one request at a time.
+      Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency).
+      If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.
   "workload_type":
-    "description": |-
-      The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).
+    "description": |
+      The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is
+      "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.
+      See the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).
 github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadSize:
   "_":
+    "description": |
+      The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.
+      A single unit of provisioned concurrency can process one request at a time.
+      Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency).
+      If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.
     "enum":
       - |-
         Small
@@ -3187,26 +3129,17 @@ github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkload
         Large
 github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadType:
   "_":
+    "description": |
+      The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is
+      "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.
+      See the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).
     "enum":
       - |-
         CPU
-      - |-
-        GPU_MEDIUM
       - |-
         GPU_SMALL
-      - |-
-        GPU_LARGE
-      - |-
-        MULTIGPU_MEDIUM
-github.com/databricks/databricks-sdk-go/service/serving.ServingModelWorkloadType:
-  "_":
-    "enum":
-      - |-
-        CPU
       - |-
         GPU_MEDIUM
-      - |-
-        GPU_SMALL
       - |-
         GPU_LARGE
       - |-
diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml
index 323432fa3..120a12543 100644
--- a/bundle/internal/schema/annotations_openapi_overrides.yml
+++ b/bundle/internal/schema/annotations_openapi_overrides.yml
@@ -197,14 +197,3 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger:
   "manual":
     "description": |-
       PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput:
-  "entity_version":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput:
-  "model_name":
-    "description": |-
-      PLACEHOLDER
-  "model_version":
-    "description": |-
-      PLACEHOLDER
diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json
index 17a621ba0..4a3b56814 100644
--- a/bundle/schema/jsonschema.json
+++ b/bundle/schema/jsonschema.json
@@ -546,7 +546,7 @@
                     "type": "object",
                     "properties": {
                       "ai_gateway": {
-                        "description": "The AI Gateway configuration for the serving endpoint. NOTE: Only external model and provisioned throughput endpoints are currently supported.",
+                        "description": "The AI Gateway configuration for the serving endpoint. NOTE: only external model endpoints are supported as of now.",
                         "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayConfig"
                       },
                       "config": {
@@ -554,7 +554,7 @@
                         "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.EndpointCoreConfigInput"
                       },
                       "name": {
-                        "description": "The name of the serving endpoint. This field is required and must be unique across a Databricks workspace.\nAn endpoint name can consist of alphanumeric characters, dashes, and underscores.",
+                        "description": "The name of the serving endpoint. This field is required and must be unique across a Databricks workspace.\nAn endpoint name can consist of alphanumeric characters, dashes, and underscores.\n",
                         "$ref": "#/$defs/string"
                       },
                       "permissions": {
@@ -575,6 +575,7 @@
                     },
                     "additionalProperties": false,
                     "required": [
+                      "config",
                       "name"
                     ]
                   },
@@ -4141,10 +4142,6 @@
                     "parameters": {
                       "description": "Parameters passed to the main method.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.",
                       "$ref": "#/$defs/slice/string"
-                    },
-                    "run_as_repl": {
-                      "description": "Deprecated. A value of `false` is no longer supported.",
-                      "$ref": "#/$defs/bool"
                     }
                   },
                   "additionalProperties": false
@@ -5505,11 +5502,11 @@
                   "type": "object",
                   "properties": {
                     "ai21labs_api_key": {
-                      "description": "The Databricks secret key reference for an AI21 Labs API key. If you\nprefer to paste your API key directly, see `ai21labs_api_key_plaintext`.\nYou must provide an API key using one of the following fields:\n`ai21labs_api_key` or `ai21labs_api_key_plaintext`.",
+                      "description": "The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "ai21labs_api_key_plaintext": {
-                      "description": "An AI21 Labs API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `ai21labs_api_key`. You\nmust provide an API key using one of the following fields:\n`ai21labs_api_key` or `ai21labs_api_key_plaintext`.",
+                      "description": "An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -5531,7 +5528,7 @@
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrails"
                     },
                     "inference_table_config": {
-                      "description": "Configuration for payload logging using inference tables.\nUse these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.",
+                      "description": "Configuration for payload logging using inference tables. Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayInferenceTableConfig"
                     },
                     "rate_limits": {
@@ -5539,7 +5536,7 @@
                       "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit"
                     },
                     "usage_tracking_config": {
-                      "description": "Configuration to enable usage tracking using system tables.\nThese tables allow you to monitor operational usage on endpoints and their associated costs.",
+                      "description": "Configuration to enable usage tracking using system tables. These tables allow you to monitor operational usage on endpoints and their associated costs.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayUsageTrackingConfig"
                     }
                   },
@@ -5557,7 +5554,7 @@
                   "type": "object",
                   "properties": {
                     "invalid_keywords": {
-                      "description": "List of invalid keywords.\nAI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.",
+                      "description": "List of invalid keywords. AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.",
                       "$ref": "#/$defs/slice/string"
                     },
                     "pii": {
@@ -5569,7 +5566,7 @@
                       "$ref": "#/$defs/bool"
                     },
                     "valid_topics": {
-                      "description": "The list of allowed topics.\nGiven a chat request, this guardrail flags the request if its topic is not in the allowed topics.",
+                      "description": "The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics.",
                       "$ref": "#/$defs/slice/string"
                     }
                   },
@@ -5587,11 +5584,14 @@
                   "type": "object",
                   "properties": {
                     "behavior": {
-                      "description": "Configuration for input guardrail filters.",
+                      "description": "Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehaviorBehavior"
                     }
                   },
-                  "additionalProperties": false
+                  "additionalProperties": false,
+                  "required": [
+                    "behavior"
+                  ]
                 },
                 {
                   "type": "string",
@@ -5603,6 +5603,7 @@
               "oneOf": [
                 {
                   "type": "string",
+                  "description": "Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.",
                   "enum": [
                     "NONE",
                     "BLOCK"
@@ -5642,7 +5643,7 @@
                   "type": "object",
                   "properties": {
                     "catalog_name": {
-                      "description": "The name of the catalog in Unity Catalog. Required when enabling inference tables.\nNOTE: On update, you have to disable inference table first in order to change the catalog name.",
+                      "description": "The name of the catalog in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the catalog name.",
                       "$ref": "#/$defs/string"
                     },
                     "enabled": {
@@ -5650,11 +5651,11 @@
                       "$ref": "#/$defs/bool"
                     },
                     "schema_name": {
-                      "description": "The name of the schema in Unity Catalog. Required when enabling inference tables.\nNOTE: On update, you have to disable inference table first in order to change the schema name.",
+                      "description": "The name of the schema in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the schema name.",
                       "$ref": "#/$defs/string"
                     },
                     "table_name_prefix": {
-                      "description": "The prefix of the table in Unity Catalog.\nNOTE: On update, you have to disable inference table first in order to change the prefix name.",
+                      "description": "The prefix of the table in Unity Catalog. NOTE: On update, you have to disable inference table first in order to change the prefix name.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -5673,10 +5674,10 @@
                   "properties": {
                     "calls": {
                       "description": "Used to specify how many calls are allowed for a key within the renewal_period.",
-                      "$ref": "#/$defs/int64"
+                      "$ref": "#/$defs/int"
                     },
                     "key": {
-                      "description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported,\nwith 'endpoint' being the default if not specified.",
+                      "description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitKey"
                     },
                     "renewal_period": {
@@ -5700,6 +5701,7 @@
               "oneOf": [
                 {
                   "type": "string",
+                  "description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.",
                   "enum": [
                     "user",
                     "endpoint"
@@ -5715,6 +5717,7 @@
               "oneOf": [
                 {
                   "type": "string",
+                  "description": "Renewal period field for a rate limit. Currently, only 'minute' is supported.",
                   "enum": [
                     "minute"
                   ]
@@ -5749,11 +5752,11 @@
                   "type": "object",
                   "properties": {
                     "aws_access_key_id": {
-                      "description": "The Databricks secret key reference for an AWS access key ID with\npermissions to interact with Bedrock services. If you prefer to paste\nyour API key directly, see `aws_access_key_id_plaintext`. You must provide an API\nkey using one of the following fields: `aws_access_key_id` or\n`aws_access_key_id_plaintext`.",
+                      "description": "The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "aws_access_key_id_plaintext": {
-                      "description": "An AWS access key ID with permissions to interact with Bedrock services\nprovided as a plaintext string. If you prefer to reference your key using\nDatabricks Secrets, see `aws_access_key_id`. You must provide an API key\nusing one of the following fields: `aws_access_key_id` or\n`aws_access_key_id_plaintext`.",
+                      "description": "An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "aws_region": {
@@ -5761,15 +5764,15 @@
                       "$ref": "#/$defs/string"
                     },
                     "aws_secret_access_key": {
-                      "description": "The Databricks secret key reference for an AWS secret access key paired\nwith the access key ID, with permissions to interact with Bedrock\nservices. If you prefer to paste your API key directly, see\n`aws_secret_access_key_plaintext`. You must provide an API key using one\nof the following fields: `aws_secret_access_key` or\n`aws_secret_access_key_plaintext`.",
+                      "description": "The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "aws_secret_access_key_plaintext": {
-                      "description": "An AWS secret access key paired with the access key ID, with permissions\nto interact with Bedrock services provided as a plaintext string. If you\nprefer to reference your key using Databricks Secrets, see\n`aws_secret_access_key`. You must provide an API key using one of the\nfollowing fields: `aws_secret_access_key` or\n`aws_secret_access_key_plaintext`.",
+                      "description": "An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "bedrock_provider": {
-                      "description": "The underlying provider in Amazon Bedrock. Supported values (case\ninsensitive) include: Anthropic, Cohere, AI21Labs, Amazon.",
+                      "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfigBedrockProvider"
                     }
                   },
@@ -5789,6 +5792,7 @@
               "oneOf": [
                 {
                   "type": "string",
+                  "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.",
                   "enum": [
                     "anthropic",
                     "cohere",
@@ -5808,11 +5812,11 @@
                   "type": "object",
                   "properties": {
                     "anthropic_api_key": {
-                      "description": "The Databricks secret key reference for an Anthropic API key. If you\nprefer to paste your API key directly, see `anthropic_api_key_plaintext`.\nYou must provide an API key using one of the following fields:\n`anthropic_api_key` or `anthropic_api_key_plaintext`.",
+                      "description": "The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "anthropic_api_key_plaintext": {
-                      "description": "The Anthropic API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `anthropic_api_key`. You\nmust provide an API key using one of the following fields:\n`anthropic_api_key` or `anthropic_api_key_plaintext`.",
+                      "description": "The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -5860,15 +5864,15 @@
                   "type": "object",
                   "properties": {
                     "cohere_api_base": {
-                      "description": "This is an optional field to provide a customized base URL for the Cohere\nAPI. If left unspecified, the standard Cohere base URL is used.",
+                      "description": "This is an optional field to provide a customized base URL for the Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n",
                       "$ref": "#/$defs/string"
                     },
                     "cohere_api_key": {
-                      "description": "The Databricks secret key reference for a Cohere API key. If you prefer\nto paste your API key directly, see `cohere_api_key_plaintext`. You must\nprovide an API key using one of the following fields: `cohere_api_key` or\n`cohere_api_key_plaintext`.",
+                      "description": "The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "cohere_api_key_plaintext": {
-                      "description": "The Cohere API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `cohere_api_key`. You\nmust provide an API key using one of the following fields:\n`cohere_api_key` or `cohere_api_key_plaintext`.",
+                      "description": "The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -5886,15 +5890,15 @@
                   "type": "object",
                   "properties": {
                     "databricks_api_token": {
-                      "description": "The Databricks secret key reference for a Databricks API token that\ncorresponds to a user or service principal with Can Query access to the\nmodel serving endpoint pointed to by this external model. If you prefer\nto paste your API key directly, see `databricks_api_token_plaintext`. You\nmust provide an API key using one of the following fields:\n`databricks_api_token` or `databricks_api_token_plaintext`.",
+                      "description": "The Databricks secret key reference for a Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model.\nIf you prefer to paste your API key directly, see `databricks_api_token_plaintext`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n",
                       "$ref": "#/$defs/string"
                     },
                     "databricks_api_token_plaintext": {
-                      "description": "The Databricks API token that corresponds to a user or service principal\nwith Can Query access to the model serving endpoint pointed to by this\nexternal model provided as a plaintext string. If you prefer to reference\nyour key using Databricks Secrets, see `databricks_api_token`. You must\nprovide an API key using one of the following fields:\n`databricks_api_token` or `databricks_api_token_plaintext`.",
+                      "description": "The Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `databricks_api_token`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n",
                       "$ref": "#/$defs/string"
                     },
                     "databricks_workspace_url": {
-                      "description": "The URL of the Databricks workspace containing the model serving endpoint\npointed to by this external model.",
+                      "description": "The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model.\n",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -5915,19 +5919,19 @@
                   "type": "object",
                   "properties": {
                     "auto_capture_config": {
-                      "description": "Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog.\nNote: this field is deprecated for creating new provisioned throughput endpoints,\nor updating existing provisioned throughput endpoints that never have inference table configured;\nin these cases please use AI Gateway to manage inference tables.",
+                      "description": "Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AutoCaptureConfigInput"
                     },
                     "served_entities": {
-                      "description": "The list of served entities under the serving endpoint config.",
+                      "description": "A list of served entities for the endpoint to serve. A serving endpoint can have up to 15 served entities.",
                       "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput"
                     },
                     "served_models": {
-                      "description": "(Deprecated, use served_entities instead) The list of served models under the serving endpoint config.",
+                      "description": "(Deprecated, use served_entities instead) A list of served models for the endpoint to serve. A serving endpoint can have up to 15 served models.",
                       "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput"
                     },
                     "traffic_config": {
-                      "description": "The traffic configuration associated with the serving endpoint config.",
+                      "description": "The traffic config defining how invocations to the serving endpoint should be routed.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.TrafficConfig"
                     }
                   },
@@ -6006,7 +6010,7 @@
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.PaLmConfig"
                     },
                     "provider": {
-                      "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic', 'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.",
+                      "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ExternalModelProvider"
                     },
                     "task": {
@@ -6031,6 +6035,7 @@
               "oneOf": [
                 {
                   "type": "string",
+                  "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n",
                   "enum": [
                     "ai21labs",
                     "anthropic",
@@ -6054,27 +6059,23 @@
                   "type": "object",
                   "properties": {
                     "private_key": {
-                      "description": "The Databricks secret key reference for a private key for the service\naccount which has access to the Google Cloud Vertex AI Service. See [Best\npractices for managing service account keys]. If you prefer to paste your\nAPI key directly, see `private_key_plaintext`. You must provide an API\nkey using one of the following fields: `private_key` or\n`private_key_plaintext`\n\n[Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys",
+                      "description": "The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`",
                       "$ref": "#/$defs/string"
                     },
                     "private_key_plaintext": {
-                      "description": "The private key for the service account which has access to the Google\nCloud Vertex AI Service provided as a plaintext secret. See [Best\npractices for managing service account keys]. If you prefer to reference\nyour key using Databricks Secrets, see `private_key`. You must provide an\nAPI key using one of the following fields: `private_key` or\n`private_key_plaintext`.\n\n[Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys",
+                      "description": "The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "project_id": {
-                      "description": "This is the Google Cloud project id that the service account is\nassociated with.",
+                      "description": "This is the Google Cloud project id that the service account is associated with.",
                       "$ref": "#/$defs/string"
                     },
                     "region": {
-                      "description": "This is the region for the Google Cloud Vertex AI Service. See [supported\nregions] for more details. Some models are only available in specific\nregions.\n\n[supported regions]: https://cloud.google.com/vertex-ai/docs/general/locations",
+                      "description": "This is the region for the Google Cloud Vertex AI Service. See [supported regions](https://cloud.google.com/vertex-ai/docs/general/locations) for more details. Some models are only available in specific regions.",
                       "$ref": "#/$defs/string"
                     }
                   },
-                  "additionalProperties": false,
-                  "required": [
-                    "project_id",
-                    "region"
-                  ]
+                  "additionalProperties": false
                 },
                 {
                   "type": "string",
@@ -6086,50 +6087,49 @@
               "oneOf": [
                 {
                   "type": "object",
-                  "description": "Configs needed to create an OpenAI model route.",
                   "properties": {
                     "microsoft_entra_client_id": {
-                      "description": "This field is only required for Azure AD OpenAI and is the Microsoft\nEntra Client ID.",
+                      "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.\n",
                       "$ref": "#/$defs/string"
                     },
                     "microsoft_entra_client_secret": {
-                      "description": "The Databricks secret key reference for a client secret used for\nMicrosoft Entra ID authentication. If you prefer to paste your client\nsecret directly, see `microsoft_entra_client_secret_plaintext`. You must\nprovide an API key using one of the following fields:\n`microsoft_entra_client_secret` or\n`microsoft_entra_client_secret_plaintext`.",
+                      "description": "The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication.\nIf you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n",
                       "$ref": "#/$defs/string"
                     },
                     "microsoft_entra_client_secret_plaintext": {
-                      "description": "The client secret used for Microsoft Entra ID authentication provided as\na plaintext string. If you prefer to reference your key using Databricks\nSecrets, see `microsoft_entra_client_secret`. You must provide an API key\nusing one of the following fields: `microsoft_entra_client_secret` or\n`microsoft_entra_client_secret_plaintext`.",
+                      "description": "The client secret used for Microsoft Entra ID authentication provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n",
                       "$ref": "#/$defs/string"
                     },
                     "microsoft_entra_tenant_id": {
-                      "description": "This field is only required for Azure AD OpenAI and is the Microsoft\nEntra Tenant ID.",
+                      "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.\n",
                       "$ref": "#/$defs/string"
                     },
                     "openai_api_base": {
-                      "description": "This is a field to provide a customized base URl for the OpenAI API. For\nAzure OpenAI, this field is required, and is the base URL for the Azure\nOpenAI API service provided by Azure. For other OpenAI API types, this\nfield is optional, and if left unspecified, the standard OpenAI base URL\nis used.",
+                      "description": "This is a field to provide a customized base URl for the OpenAI API.\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\nFor other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used.\n",
                       "$ref": "#/$defs/string"
                     },
                     "openai_api_key": {
-                      "description": "The Databricks secret key reference for an OpenAI API key using the\nOpenAI or Azure service. If you prefer to paste your API key directly,\nsee `openai_api_key_plaintext`. You must provide an API key using one of\nthe following fields: `openai_api_key` or `openai_api_key_plaintext`.",
+                      "description": "The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "openai_api_key_plaintext": {
-                      "description": "The OpenAI API key using the OpenAI or Azure service provided as a\nplaintext string. If you prefer to reference your key using Databricks\nSecrets, see `openai_api_key`. You must provide an API key using one of\nthe following fields: `openai_api_key` or `openai_api_key_plaintext`.",
+                      "description": "The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "openai_api_type": {
-                      "description": "This is an optional field to specify the type of OpenAI API to use. For\nAzure OpenAI, this field is required, and adjust this parameter to\nrepresent the preferred security access validation protocol. For access\ntoken validation, use azure. For authentication using Azure Active\nDirectory (Azure AD) use, azuread.",
+                      "description": "This is an optional field to specify the type of OpenAI API to use.\nFor Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security\naccess validation protocol. For access token validation, use azure. For authentication using Azure Active\nDirectory (Azure AD) use, azuread.\n",
                       "$ref": "#/$defs/string"
                     },
                     "openai_api_version": {
-                      "description": "This is an optional field to specify the OpenAI API version. For Azure\nOpenAI, this field is required, and is the version of the Azure OpenAI\nservice to utilize, specified by a date.",
+                      "description": "This is an optional field to specify the OpenAI API version.\nFor Azure OpenAI, this field is required, and is the version of the Azure OpenAI service to\nutilize, specified by a date.\n",
                       "$ref": "#/$defs/string"
                     },
                     "openai_deployment_name": {
-                      "description": "This field is only required for Azure OpenAI and is the name of the\ndeployment resource for the Azure OpenAI service.",
+                      "description": "This field is only required for Azure OpenAI and is the name of the deployment resource for the\nAzure OpenAI service.\n",
                       "$ref": "#/$defs/string"
                     },
                     "openai_organization": {
-                      "description": "This is an optional field to specify the organization in OpenAI or Azure\nOpenAI.",
+                      "description": "This is an optional field to specify the organization in OpenAI or Azure OpenAI.\n",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -6147,11 +6147,11 @@
                   "type": "object",
                   "properties": {
                     "palm_api_key": {
-                      "description": "The Databricks secret key reference for a PaLM API key. If you prefer to\npaste your API key directly, see `palm_api_key_plaintext`. You must\nprovide an API key using one of the following fields: `palm_api_key` or\n`palm_api_key_plaintext`.",
+                      "description": "The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "palm_api_key_plaintext": {
-                      "description": "The PaLM API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `palm_api_key`. You must\nprovide an API key using one of the following fields: `palm_api_key` or\n`palm_api_key_plaintext`.",
+                      "description": "The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -6170,7 +6170,7 @@
                   "properties": {
                     "calls": {
                       "description": "Used to specify how many calls are allowed for a key within the renewal_period.",
-                      "$ref": "#/$defs/int64"
+                      "$ref": "#/$defs/int"
                     },
                     "key": {
                       "description": "Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.",
@@ -6197,6 +6197,7 @@
               "oneOf": [
                 {
                   "type": "string",
+                  "description": "Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.",
                   "enum": [
                     "user",
                     "endpoint"
@@ -6212,6 +6213,7 @@
               "oneOf": [
                 {
                   "type": "string",
+                  "description": "Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported.",
                   "enum": [
                     "minute"
                   ]
@@ -6254,18 +6256,19 @@
                   "type": "object",
                   "properties": {
                     "entity_name": {
-                      "description": "The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC), or a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of **catalog_name.schema_name.model_name**.",
+                      "description": "The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC),\nor a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of\n__catalog_name__.__schema_name__.__model_name__.\n",
                       "$ref": "#/$defs/string"
                     },
                     "entity_version": {
+                      "description": "The version of the model in Databricks Model Registry to be served or empty if the entity is a FEATURE_SPEC.",
                       "$ref": "#/$defs/string"
                     },
                     "environment_vars": {
-                      "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`",
+                      "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity.\nNote: this is an experimental feature and subject to change. \nExample entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`",
                       "$ref": "#/$defs/map/string"
                     },
                     "external_model": {
-                      "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same.",
+                      "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)\ncan be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model,\nit cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later.\nThe task type of all external models within an endpoint must be the same.\n",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ExternalModel"
                     },
                     "instance_profile_arn": {
@@ -6281,7 +6284,7 @@
                       "$ref": "#/$defs/int"
                     },
                     "name": {
-                      "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.",
+                      "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores.\nIf not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other\nentities, it defaults to \u003centity-name\u003e-\u003centity-version\u003e.\n",
                       "$ref": "#/$defs/string"
                     },
                     "scale_to_zero_enabled": {
@@ -6289,12 +6292,12 @@
                       "$ref": "#/$defs/bool"
                     },
                     "workload_size": {
-                      "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.",
+                      "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.\n",
                       "$ref": "#/$defs/string"
                     },
                     "workload_type": {
-                      "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is \"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).",
-                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServingModelWorkloadType"
+                      "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n",
+                      "$ref": "#/$defs/string"
                     }
                   },
                   "additionalProperties": false
@@ -6311,11 +6314,11 @@
                   "type": "object",
                   "properties": {
                     "environment_vars": {
-                      "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`",
+                      "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this model.\nNote: this is an experimental feature and subject to change. \nExample model environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`",
                       "$ref": "#/$defs/map/string"
                     },
                     "instance_profile_arn": {
-                      "description": "ARN of the instance profile that the served entity uses to access AWS resources.",
+                      "description": "ARN of the instance profile that the served model will use to access AWS resources.",
                       "$ref": "#/$defs/string"
                     },
                     "max_provisioned_throughput": {
@@ -6327,25 +6330,27 @@
                       "$ref": "#/$defs/int"
                     },
                     "model_name": {
+                      "description": "The name of the model in Databricks Model Registry to be served or if the model resides in Unity Catalog, the full name of model,\nin the form of __catalog_name__.__schema_name__.__model_name__.\n",
                       "$ref": "#/$defs/string"
                     },
                     "model_version": {
+                      "description": "The version of the model in Databricks Model Registry or Unity Catalog to be served.",
                       "$ref": "#/$defs/string"
                     },
                     "name": {
-                      "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.",
+                      "description": "The name of a served model. It must be unique across an endpoint. If not specified, this field will default to \u003cmodel-name\u003e-\u003cmodel-version\u003e.\nA served model name can consist of alphanumeric characters, dashes, and underscores.\n",
                       "$ref": "#/$defs/string"
                     },
                     "scale_to_zero_enabled": {
-                      "description": "Whether the compute resources for the served entity should scale down to zero.",
+                      "description": "Whether the compute resources for the served model should scale down to zero.",
                       "$ref": "#/$defs/bool"
                     },
                     "workload_size": {
-                      "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.",
+                      "description": "The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.\n",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadSize"
                     },
                     "workload_type": {
-                      "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is \"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).",
+                      "description": "The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadType"
                     }
                   },
@@ -6366,6 +6371,7 @@
               "oneOf": [
                 {
                   "type": "string",
+                  "description": "The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.\n",
                   "enum": [
                     "Small",
                     "Medium",
@@ -6382,28 +6388,11 @@
               "oneOf": [
                 {
                   "type": "string",
+                  "description": "The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n",
                   "enum": [
                     "CPU",
-                    "GPU_MEDIUM",
                     "GPU_SMALL",
-                    "GPU_LARGE",
-                    "MULTIGPU_MEDIUM"
-                  ]
-                },
-                {
-                  "type": "string",
-                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
-                }
-              ]
-            },
-            "serving.ServingModelWorkloadType": {
-              "oneOf": [
-                {
-                  "type": "string",
-                  "enum": [
-                    "CPU",
                     "GPU_MEDIUM",
-                    "GPU_SMALL",
                     "GPU_LARGE",
                     "MULTIGPU_MEDIUM"
                   ]
diff --git a/cmd/account/custom-app-integration/custom-app-integration.go b/cmd/account/custom-app-integration/custom-app-integration.go
index 43e458bc6..1eec1018e 100755
--- a/cmd/account/custom-app-integration/custom-app-integration.go
+++ b/cmd/account/custom-app-integration/custom-app-integration.go
@@ -307,7 +307,6 @@ func newUpdate() *cobra.Command {
 	cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
 	// TODO: array: redirect_urls
-	// TODO: array: scopes
 	// TODO: complex arg: token_access_policy
 
 	cmd.Use = "update INTEGRATION_ID"
diff --git a/cmd/api/api.go b/cmd/api/api.go
index fad8a026f..c3a3eb0b6 100644
--- a/cmd/api/api.go
+++ b/cmd/api/api.go
@@ -62,7 +62,7 @@ func makeCommand(method string) *cobra.Command {
 
 			var response any
 			headers := map[string]string{"Content-Type": "application/json"}
-			err = api.Do(cmd.Context(), method, path, headers, nil, request, &response)
+			err = api.Do(cmd.Context(), method, path, headers, request, &response)
 			if err != nil {
 				return err
 			}
diff --git a/cmd/workspace/access-control/access-control.go b/cmd/workspace/access-control/access-control.go
deleted file mode 100755
index 7668265fb..000000000
--- a/cmd/workspace/access-control/access-control.go
+++ /dev/null
@@ -1,109 +0,0 @@
-// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT.
-
-package access_control
-
-import (
-	"fmt"
-
-	"github.com/databricks/cli/cmd/root"
-	"github.com/databricks/cli/libs/cmdio"
-	"github.com/databricks/cli/libs/flags"
-	"github.com/databricks/databricks-sdk-go/service/iam"
-	"github.com/spf13/cobra"
-)
-
-// Slice with functions to override default command behavior.
-// Functions can be added from the `init()` function in manually curated files in this directory.
-var cmdOverrides []func(*cobra.Command)
-
-func New() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:     "access-control",
-		Short:   `Rule based Access Control for Databricks Resources.`,
-		Long:    `Rule based Access Control for Databricks Resources.`,
-		GroupID: "iam",
-		Annotations: map[string]string{
-			"package": "iam",
-		},
-
-		// This service is being previewed; hide from help output.
-		Hidden: true,
-	}
-
-	// Add methods
-	cmd.AddCommand(newCheckPolicy())
-
-	// Apply optional overrides to this command.
-	for _, fn := range cmdOverrides {
-		fn(cmd)
-	}
-
-	return cmd
-}
-
-// start check-policy command
-
-// Slice with functions to override default command behavior.
-// Functions can be added from the `init()` function in manually curated files in this directory.
-var checkPolicyOverrides []func(
-	*cobra.Command,
-	*iam.CheckPolicyRequest,
-)
-
-func newCheckPolicy() *cobra.Command {
-	cmd := &cobra.Command{}
-
-	var checkPolicyReq iam.CheckPolicyRequest
-	var checkPolicyJson flags.JsonFlag
-
-	// TODO: short flags
-	cmd.Flags().Var(&checkPolicyJson, "json", `either inline JSON string or @path/to/file.json with request body`)
-
-	// TODO: complex arg: resource_info
-
-	cmd.Use = "check-policy"
-	cmd.Short = `Check access policy to a resource.`
-	cmd.Long = `Check access policy to a resource.`
-
-	cmd.Annotations = make(map[string]string)
-
-	cmd.PreRunE = root.MustWorkspaceClient
-	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
-		ctx := cmd.Context()
-		w := root.WorkspaceClient(ctx)
-
-		if cmd.Flags().Changed("json") {
-			diags := checkPolicyJson.Unmarshal(&checkPolicyReq)
-			if diags.HasError() {
-				return diags.Error()
-			}
-			if len(diags) > 0 {
-				err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags)
-				if err != nil {
-					return err
-				}
-			}
-		} else {
-			return fmt.Errorf("please provide command input in JSON format by specifying the --json flag")
-		}
-
-		response, err := w.AccessControl.CheckPolicy(ctx, checkPolicyReq)
-		if err != nil {
-			return err
-		}
-		return cmdio.Render(ctx, response)
-	}
-
-	// Disable completions since they are not applicable.
-	// Can be overridden by manual implementation in `override.go`.
-	cmd.ValidArgsFunction = cobra.NoFileCompletions
-
-	// Apply optional overrides to this command.
-	for _, fn := range checkPolicyOverrides {
-		fn(cmd, &checkPolicyReq)
-	}
-
-	return cmd
-}
-
-// end service AccessControl
diff --git a/cmd/workspace/cmd.go b/cmd/workspace/cmd.go
index c447bd736..f07d0cf76 100755
--- a/cmd/workspace/cmd.go
+++ b/cmd/workspace/cmd.go
@@ -3,7 +3,6 @@
 package workspace
 
 import (
-	access_control "github.com/databricks/cli/cmd/workspace/access-control"
 	alerts "github.com/databricks/cli/cmd/workspace/alerts"
 	alerts_legacy "github.com/databricks/cli/cmd/workspace/alerts-legacy"
 	apps "github.com/databricks/cli/cmd/workspace/apps"
@@ -97,7 +96,6 @@ import (
 func All() []*cobra.Command {
 	var out []*cobra.Command
 
-	out = append(out, access_control.New())
 	out = append(out, alerts.New())
 	out = append(out, alerts_legacy.New())
 	out = append(out, apps.New())
diff --git a/cmd/workspace/providers/providers.go b/cmd/workspace/providers/providers.go
index 4d6262cff..504beac5e 100755
--- a/cmd/workspace/providers/providers.go
+++ b/cmd/workspace/providers/providers.go
@@ -64,7 +64,7 @@ func newCreate() *cobra.Command {
 	cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
 	cmd.Flags().StringVar(&createReq.Comment, "comment", createReq.Comment, `Description about the provider.`)
-	cmd.Flags().StringVar(&createReq.RecipientProfileStr, "recipient-profile-str", createReq.RecipientProfileStr, `This field is required when the __authentication_type__ is **TOKEN**, **OAUTH_CLIENT_CREDENTIALS** or not provided.`)
+	cmd.Flags().StringVar(&createReq.RecipientProfileStr, "recipient-profile-str", createReq.RecipientProfileStr, `This field is required when the __authentication_type__ is **TOKEN** or not provided.`)
 
 	cmd.Use = "create NAME AUTHENTICATION_TYPE"
 	cmd.Short = `Create an auth provider.`
@@ -430,7 +430,7 @@ func newUpdate() *cobra.Command {
 	cmd.Flags().StringVar(&updateReq.Comment, "comment", updateReq.Comment, `Description about the provider.`)
 	cmd.Flags().StringVar(&updateReq.NewName, "new-name", updateReq.NewName, `New name for the provider.`)
 	cmd.Flags().StringVar(&updateReq.Owner, "owner", updateReq.Owner, `Username of Provider owner.`)
-	cmd.Flags().StringVar(&updateReq.RecipientProfileStr, "recipient-profile-str", updateReq.RecipientProfileStr, `This field is required when the __authentication_type__ is **TOKEN**, **OAUTH_CLIENT_CREDENTIALS** or not provided.`)
+	cmd.Flags().StringVar(&updateReq.RecipientProfileStr, "recipient-profile-str", updateReq.RecipientProfileStr, `This field is required when the __authentication_type__ is **TOKEN** or not provided.`)
 
 	cmd.Use = "update NAME"
 	cmd.Short = `Update a provider.`
diff --git a/cmd/workspace/recipients/recipients.go b/cmd/workspace/recipients/recipients.go
index 6d6ce42f1..56abd2014 100755
--- a/cmd/workspace/recipients/recipients.go
+++ b/cmd/workspace/recipients/recipients.go
@@ -91,7 +91,7 @@ func newCreate() *cobra.Command {
 	cmd.Long = `Create a share recipient.
   
   Creates a new recipient with the delta sharing authentication type in the
-  metastore. The caller must be a metastore admin or have the
+  metastore. The caller must be a metastore admin or has the
   **CREATE_RECIPIENT** privilege on the metastore.
 
   Arguments:
@@ -186,16 +186,28 @@ func newDelete() *cobra.Command {
 
 	cmd.Annotations = make(map[string]string)
 
-	cmd.Args = func(cmd *cobra.Command, args []string) error {
-		check := root.ExactArgs(1)
-		return check(cmd, args)
-	}
-
 	cmd.PreRunE = root.MustWorkspaceClient
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
 
+		if len(args) == 0 {
+			promptSpinner := cmdio.Spinner(ctx)
+			promptSpinner <- "No NAME argument specified. Loading names for Recipients drop-down."
+			names, err := w.Recipients.RecipientInfoNameToMetastoreIdMap(ctx, sharing.ListRecipientsRequest{})
+			close(promptSpinner)
+			if err != nil {
+				return fmt.Errorf("failed to load names for Recipients drop-down. Please manually specify required arguments. Original error: %w", err)
+			}
+			id, err := cmdio.Select(ctx, names, "Name of the recipient")
+			if err != nil {
+				return err
+			}
+			args = append(args, id)
+		}
+		if len(args) != 1 {
+			return fmt.Errorf("expected to have name of the recipient")
+		}
 		deleteReq.Name = args[0]
 
 		err = w.Recipients.Delete(ctx, deleteReq)
@@ -246,16 +258,28 @@ func newGet() *cobra.Command {
 
 	cmd.Annotations = make(map[string]string)
 
-	cmd.Args = func(cmd *cobra.Command, args []string) error {
-		check := root.ExactArgs(1)
-		return check(cmd, args)
-	}
-
 	cmd.PreRunE = root.MustWorkspaceClient
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
 
+		if len(args) == 0 {
+			promptSpinner := cmdio.Spinner(ctx)
+			promptSpinner <- "No NAME argument specified. Loading names for Recipients drop-down."
+			names, err := w.Recipients.RecipientInfoNameToMetastoreIdMap(ctx, sharing.ListRecipientsRequest{})
+			close(promptSpinner)
+			if err != nil {
+				return fmt.Errorf("failed to load names for Recipients drop-down. Please manually specify required arguments. Original error: %w", err)
+			}
+			id, err := cmdio.Select(ctx, names, "Name of the recipient")
+			if err != nil {
+				return err
+			}
+			args = append(args, id)
+		}
+		if len(args) != 1 {
+			return fmt.Errorf("expected to have name of the recipient")
+		}
 		getReq.Name = args[0]
 
 		response, err := w.Recipients.Get(ctx, getReq)
@@ -360,7 +384,7 @@ func newRotateToken() *cobra.Command {
   the provided token info. The caller must be the owner of the recipient.
 
   Arguments:
-    NAME: The name of the Recipient.
+    NAME: The name of the recipient.
     EXISTING_TOKEN_EXPIRE_IN_SECONDS: The expiration time of the bearer token in ISO 8601 format. This will set
       the expiration_time of existing token only to a smaller timestamp, it
       cannot extend the expiration_time. Use 0 to expire the existing token
@@ -455,16 +479,28 @@ func newSharePermissions() *cobra.Command {
 
 	cmd.Annotations = make(map[string]string)
 
-	cmd.Args = func(cmd *cobra.Command, args []string) error {
-		check := root.ExactArgs(1)
-		return check(cmd, args)
-	}
-
 	cmd.PreRunE = root.MustWorkspaceClient
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
 
+		if len(args) == 0 {
+			promptSpinner := cmdio.Spinner(ctx)
+			promptSpinner <- "No NAME argument specified. Loading names for Recipients drop-down."
+			names, err := w.Recipients.RecipientInfoNameToMetastoreIdMap(ctx, sharing.ListRecipientsRequest{})
+			close(promptSpinner)
+			if err != nil {
+				return fmt.Errorf("failed to load names for Recipients drop-down. Please manually specify required arguments. Original error: %w", err)
+			}
+			id, err := cmdio.Select(ctx, names, "The name of the Recipient")
+			if err != nil {
+				return err
+			}
+			args = append(args, id)
+		}
+		if len(args) != 1 {
+			return fmt.Errorf("expected to have the name of the recipient")
+		}
 		sharePermissionsReq.Name = args[0]
 
 		response, err := w.Recipients.SharePermissions(ctx, sharePermissionsReq)
@@ -524,11 +560,6 @@ func newUpdate() *cobra.Command {
 
 	cmd.Annotations = make(map[string]string)
 
-	cmd.Args = func(cmd *cobra.Command, args []string) error {
-		check := root.ExactArgs(1)
-		return check(cmd, args)
-	}
-
 	cmd.PreRunE = root.MustWorkspaceClient
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
@@ -546,13 +577,30 @@ func newUpdate() *cobra.Command {
 				}
 			}
 		}
+		if len(args) == 0 {
+			promptSpinner := cmdio.Spinner(ctx)
+			promptSpinner <- "No NAME argument specified. Loading names for Recipients drop-down."
+			names, err := w.Recipients.RecipientInfoNameToMetastoreIdMap(ctx, sharing.ListRecipientsRequest{})
+			close(promptSpinner)
+			if err != nil {
+				return fmt.Errorf("failed to load names for Recipients drop-down. Please manually specify required arguments. Original error: %w", err)
+			}
+			id, err := cmdio.Select(ctx, names, "Name of the recipient")
+			if err != nil {
+				return err
+			}
+			args = append(args, id)
+		}
+		if len(args) != 1 {
+			return fmt.Errorf("expected to have name of the recipient")
+		}
 		updateReq.Name = args[0]
 
-		response, err := w.Recipients.Update(ctx, updateReq)
+		err = w.Recipients.Update(ctx, updateReq)
 		if err != nil {
 			return err
 		}
-		return cmdio.Render(ctx, response)
+		return nil
 	}
 
 	// Disable completions since they are not applicable.
diff --git a/cmd/workspace/serving-endpoints/serving-endpoints.go b/cmd/workspace/serving-endpoints/serving-endpoints.go
index 034133623..cc99177c7 100755
--- a/cmd/workspace/serving-endpoints/serving-endpoints.go
+++ b/cmd/workspace/serving-endpoints/serving-endpoints.go
@@ -49,7 +49,6 @@ func New() *cobra.Command {
 	cmd.AddCommand(newGetOpenApi())
 	cmd.AddCommand(newGetPermissionLevels())
 	cmd.AddCommand(newGetPermissions())
-	cmd.AddCommand(newHttpRequest())
 	cmd.AddCommand(newList())
 	cmd.AddCommand(newLogs())
 	cmd.AddCommand(newPatch())
@@ -154,34 +153,16 @@ func newCreate() *cobra.Command {
 	cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
 	// TODO: complex arg: ai_gateway
-	// TODO: complex arg: config
 	// TODO: array: rate_limits
 	cmd.Flags().BoolVar(&createReq.RouteOptimized, "route-optimized", createReq.RouteOptimized, `Enable route optimization for the serving endpoint.`)
 	// TODO: array: tags
 
-	cmd.Use = "create NAME"
+	cmd.Use = "create"
 	cmd.Short = `Create a new serving endpoint.`
-	cmd.Long = `Create a new serving endpoint.
-
-  Arguments:
-    NAME: The name of the serving endpoint. This field is required and must be
-      unique across a Databricks workspace. An endpoint name can consist of
-      alphanumeric characters, dashes, and underscores.`
+	cmd.Long = `Create a new serving endpoint.`
 
 	cmd.Annotations = make(map[string]string)
 
-	cmd.Args = func(cmd *cobra.Command, args []string) error {
-		if cmd.Flags().Changed("json") {
-			err := root.ExactArgs(0)(cmd, args)
-			if err != nil {
-				return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'name' in your JSON input")
-			}
-			return nil
-		}
-		check := root.ExactArgs(1)
-		return check(cmd, args)
-	}
-
 	cmd.PreRunE = root.MustWorkspaceClient
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
@@ -198,9 +179,8 @@ func newCreate() *cobra.Command {
 					return err
 				}
 			}
-		}
-		if !cmd.Flags().Changed("json") {
-			createReq.Name = args[0]
+		} else {
+			return fmt.Errorf("please provide command input in JSON format by specifying the --json flag")
 		}
 
 		wait, err := w.ServingEndpoints.Create(ctx, createReq)
@@ -253,7 +233,10 @@ func newDelete() *cobra.Command {
 
 	cmd.Use = "delete NAME"
 	cmd.Short = `Delete a serving endpoint.`
-	cmd.Long = `Delete a serving endpoint.`
+	cmd.Long = `Delete a serving endpoint.
+
+  Arguments:
+    NAME: The name of the serving endpoint. This field is required.`
 
 	cmd.Annotations = make(map[string]string)
 
@@ -449,12 +432,11 @@ func newGetOpenApi() *cobra.Command {
 
 		getOpenApiReq.Name = args[0]
 
-		response, err := w.ServingEndpoints.GetOpenApi(ctx, getOpenApiReq)
+		err = w.ServingEndpoints.GetOpenApi(ctx, getOpenApiReq)
 		if err != nil {
 			return err
 		}
-		defer response.Contents.Close()
-		return cmdio.Render(ctx, response.Contents)
+		return nil
 	}
 
 	// Disable completions since they are not applicable.
@@ -586,77 +568,6 @@ func newGetPermissions() *cobra.Command {
 	return cmd
 }
 
-// start http-request command
-
-// Slice with functions to override default command behavior.
-// Functions can be added from the `init()` function in manually curated files in this directory.
-var httpRequestOverrides []func(
-	*cobra.Command,
-	*serving.ExternalFunctionRequest,
-)
-
-func newHttpRequest() *cobra.Command {
-	cmd := &cobra.Command{}
-
-	var httpRequestReq serving.ExternalFunctionRequest
-
-	// TODO: short flags
-
-	cmd.Flags().StringVar(&httpRequestReq.Headers, "headers", httpRequestReq.Headers, `Additional headers for the request.`)
-	cmd.Flags().StringVar(&httpRequestReq.Json, "json", httpRequestReq.Json, `The JSON payload to send in the request body.`)
-	cmd.Flags().StringVar(&httpRequestReq.Params, "params", httpRequestReq.Params, `Query parameters for the request.`)
-
-	cmd.Use = "http-request CONNECTION_NAME METHOD PATH"
-	cmd.Short = `Make external services call using the credentials stored in UC Connection.`
-	cmd.Long = `Make external services call using the credentials stored in UC Connection.
-
-  Arguments:
-    CONNECTION_NAME: The connection name to use. This is required to identify the external
-      connection.
-    METHOD: The HTTP method to use (e.g., 'GET', 'POST').
-    PATH: The relative path for the API endpoint. This is required.`
-
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
-	cmd.Annotations = make(map[string]string)
-
-	cmd.Args = func(cmd *cobra.Command, args []string) error {
-		check := root.ExactArgs(3)
-		return check(cmd, args)
-	}
-
-	cmd.PreRunE = root.MustWorkspaceClient
-	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
-		ctx := cmd.Context()
-		w := root.WorkspaceClient(ctx)
-
-		httpRequestReq.ConnectionName = args[0]
-		_, err = fmt.Sscan(args[1], &httpRequestReq.Method)
-		if err != nil {
-			return fmt.Errorf("invalid METHOD: %s", args[1])
-		}
-		httpRequestReq.Path = args[2]
-
-		response, err := w.ServingEndpoints.HttpRequest(ctx, httpRequestReq)
-		if err != nil {
-			return err
-		}
-		return cmdio.Render(ctx, response)
-	}
-
-	// Disable completions since they are not applicable.
-	// Can be overridden by manual implementation in `override.go`.
-	cmd.ValidArgsFunction = cobra.NoFileCompletions
-
-	// Apply optional overrides to this command.
-	for _, fn := range httpRequestOverrides {
-		fn(cmd, &httpRequestReq)
-	}
-
-	return cmd
-}
-
 // start list command
 
 // Slice with functions to override default command behavior.
@@ -938,7 +849,7 @@ func newPutAiGateway() *cobra.Command {
 	cmd.Long = `Update AI Gateway of a serving endpoint.
   
   Used to update the AI Gateway of a serving endpoint. NOTE: Only external model
-  and provisioned throughput endpoints are currently supported.
+  endpoints are currently supported.
 
   Arguments:
     NAME: The name of the serving endpoint whose AI Gateway is being updated. This
diff --git a/go.mod b/go.mod
index 4a3bf1620..0ef800d7b 100644
--- a/go.mod
+++ b/go.mod
@@ -7,7 +7,7 @@ toolchain go1.23.4
 require (
 	github.com/Masterminds/semver/v3 v3.3.1 // MIT
 	github.com/briandowns/spinner v1.23.1 // Apache 2.0
-	github.com/databricks/databricks-sdk-go v0.56.0 // Apache 2.0
+	github.com/databricks/databricks-sdk-go v0.55.0 // Apache 2.0
 	github.com/fatih/color v1.18.0 // MIT
 	github.com/google/uuid v1.6.0 // BSD-3-Clause
 	github.com/hashicorp/go-version v1.7.0 // MPL 2.0
diff --git a/go.sum b/go.sum
index b4e92c2c9..b1364cb26 100644
--- a/go.sum
+++ b/go.sum
@@ -32,8 +32,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cyphar/filepath-securejoin v0.2.5 h1:6iR5tXJ/e6tJZzzdMc1km3Sa7RRIVBKAK32O2s7AYfo=
 github.com/cyphar/filepath-securejoin v0.2.5/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
-github.com/databricks/databricks-sdk-go v0.56.0 h1:8BsqjrSLbm2ET+/SLCN8qD+v+HFvs891dzi1OaiyRfc=
-github.com/databricks/databricks-sdk-go v0.56.0/go.mod h1:JpLizplEs+up9/Z4Xf2x++o3sM9eTTWFGzIXAptKJzI=
+github.com/databricks/databricks-sdk-go v0.55.0 h1:ReziD6spzTDltM0ml80LggKo27F3oUjgTinCFDJDnak=
+github.com/databricks/databricks-sdk-go v0.55.0/go.mod h1:JpLizplEs+up9/Z4Xf2x++o3sM9eTTWFGzIXAptKJzI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
diff --git a/integration/cmd/sync/sync_test.go b/integration/cmd/sync/sync_test.go
index 88e6ed89a..632497054 100644
--- a/integration/cmd/sync/sync_test.go
+++ b/integration/cmd/sync/sync_test.go
@@ -158,7 +158,7 @@ func (a *syncTest) remoteFileContent(ctx context.Context, relativePath, expected
 
 	var res []byte
 	a.c.Eventually(func() bool {
-		err = apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, nil, &res)
+		err = apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, &res)
 		require.NoError(a.t, err)
 		actualContent := string(res)
 		return actualContent == expectedContent
diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go
index 7102b6e29..88bbadd32 100644
--- a/libs/filer/files_client.go
+++ b/libs/filer/files_client.go
@@ -148,7 +148,7 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader,
 	overwrite := slices.Contains(mode, OverwriteIfExists)
 	urlPath = fmt.Sprintf("%s?overwrite=%t", urlPath, overwrite)
 	headers := map[string]string{"Content-Type": "application/octet-stream"}
-	err = w.apiClient.Do(ctx, http.MethodPut, urlPath, headers, nil, reader, nil)
+	err = w.apiClient.Do(ctx, http.MethodPut, urlPath, headers, reader, nil)
 
 	// Return early on success.
 	if err == nil {
@@ -176,7 +176,7 @@ func (w *FilesClient) Read(ctx context.Context, name string) (io.ReadCloser, err
 	}
 
 	var reader io.ReadCloser
-	err = w.apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, nil, &reader)
+	err = w.apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, &reader)
 
 	// Return early on success.
 	if err == nil {
diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go
index f7e1b3adb..8d5148edd 100644
--- a/libs/filer/workspace_files_client.go
+++ b/libs/filer/workspace_files_client.go
@@ -106,7 +106,7 @@ func (info *wsfsFileInfo) MarshalJSON() ([]byte, error) {
 // as an interface to allow for mocking in tests.
 type apiClient interface {
 	Do(ctx context.Context, method, path string,
-		headers map[string]string, queryParams map[string]any, request, response any,
+		headers map[string]string, request, response any,
 		visitors ...func(*http.Request) error) error
 }
 
@@ -156,7 +156,7 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io
 		return err
 	}
 
-	err = w.apiClient.Do(ctx, http.MethodPost, urlPath, nil, nil, body, nil)
+	err = w.apiClient.Do(ctx, http.MethodPost, urlPath, nil, body, nil)
 
 	// Return early on success.
 	if err == nil {
@@ -341,7 +341,6 @@ func (w *WorkspaceFilesClient) Stat(ctx context.Context, name string) (fs.FileIn
 		http.MethodGet,
 		"/api/2.0/workspace/get-status",
 		nil,
-		nil,
 		map[string]string{
 			"path":               absPath,
 			"return_export_info": "true",
diff --git a/libs/filer/workspace_files_extensions_client_test.go b/libs/filer/workspace_files_extensions_client_test.go
index f9c65d6ee..9ea837fa9 100644
--- a/libs/filer/workspace_files_extensions_client_test.go
+++ b/libs/filer/workspace_files_extensions_client_test.go
@@ -17,7 +17,7 @@ type mockApiClient struct {
 }
 
 func (m *mockApiClient) Do(ctx context.Context, method, path string,
-	headers map[string]string, queryParams map[string]any, request, response any,
+	headers map[string]string, request, response any,
 	visitors ...func(*http.Request) error,
 ) error {
 	args := m.Called(ctx, method, path, headers, request, response, visitors)
diff --git a/libs/git/info.go b/libs/git/info.go
index dc4af9b6d..46e57be48 100644
--- a/libs/git/info.go
+++ b/libs/git/info.go
@@ -66,7 +66,6 @@ func fetchRepositoryInfoAPI(ctx context.Context, path string, w *databricks.Work
 		http.MethodGet,
 		apiEndpoint,
 		nil,
-		nil,
 		map[string]string{
 			"path":            path,
 			"return_git_info": "true",

From 8af9efaa621103308bd869662602b3724406c173 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Thu, 23 Jan 2025 14:58:18 +0100
Subject: [PATCH 137/247] Show an error when non-yaml files used in include
 section (#2201)

## Changes
`include` section is used only to include other bundle configuration
YAML files. If any other file type is used, raise an error and guide
users to use `sync.include` instead

## Tests
Added acceptance test

---------

Co-authored-by: Julia Crawford (Databricks) <julia.crawford@databricks.com>
---
 .../includes/non_yaml_in_include/databricks.yml |  6 ++++++
 .../includes/non_yaml_in_include/output.txt     | 10 ++++++++++
 .../bundle/includes/non_yaml_in_include/script  |  1 +
 .../bundle/includes/non_yaml_in_include/test.py |  1 +
 bundle/config/loader/process_root_includes.go   | 17 ++++++++++++++++-
 5 files changed, 34 insertions(+), 1 deletion(-)
 create mode 100644 acceptance/bundle/includes/non_yaml_in_include/databricks.yml
 create mode 100644 acceptance/bundle/includes/non_yaml_in_include/output.txt
 create mode 100644 acceptance/bundle/includes/non_yaml_in_include/script
 create mode 100644 acceptance/bundle/includes/non_yaml_in_include/test.py

diff --git a/acceptance/bundle/includes/non_yaml_in_include/databricks.yml b/acceptance/bundle/includes/non_yaml_in_include/databricks.yml
new file mode 100644
index 000000000..162bd6013
--- /dev/null
+++ b/acceptance/bundle/includes/non_yaml_in_include/databricks.yml
@@ -0,0 +1,6 @@
+bundle:
+  name: non_yaml_in_includes
+
+include:
+ - test.py
+ - resources/*.yml
diff --git a/acceptance/bundle/includes/non_yaml_in_include/output.txt b/acceptance/bundle/includes/non_yaml_in_include/output.txt
new file mode 100644
index 000000000..6006ca14e
--- /dev/null
+++ b/acceptance/bundle/includes/non_yaml_in_include/output.txt
@@ -0,0 +1,10 @@
+Error: Files in the 'include' configuration section must be YAML files.
+  in databricks.yml:5:4
+
+The file test.py in the 'include' configuration section is not a YAML file, and only YAML files are supported. To include files to sync, specify them in the 'sync.include' configuration section instead.
+
+Name: non_yaml_in_includes
+
+Found 1 error
+
+Exit code: 1
diff --git a/acceptance/bundle/includes/non_yaml_in_include/script b/acceptance/bundle/includes/non_yaml_in_include/script
new file mode 100644
index 000000000..72555b332
--- /dev/null
+++ b/acceptance/bundle/includes/non_yaml_in_include/script
@@ -0,0 +1 @@
+$CLI bundle validate
diff --git a/acceptance/bundle/includes/non_yaml_in_include/test.py b/acceptance/bundle/includes/non_yaml_in_include/test.py
new file mode 100644
index 000000000..44159b395
--- /dev/null
+++ b/acceptance/bundle/includes/non_yaml_in_include/test.py
@@ -0,0 +1 @@
+print("Hello world")
diff --git a/bundle/config/loader/process_root_includes.go b/bundle/config/loader/process_root_includes.go
index c608a3de6..198095742 100644
--- a/bundle/config/loader/process_root_includes.go
+++ b/bundle/config/loader/process_root_includes.go
@@ -2,6 +2,7 @@ package loader
 
 import (
 	"context"
+	"fmt"
 	"path/filepath"
 	"slices"
 	"strings"
@@ -36,6 +37,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.
 	// Maintain list of files in order of files being loaded.
 	// This is stored in the bundle configuration for observability.
 	var files []string
+	var diags diag.Diagnostics
 
 	// For each glob, find all files to load.
 	// Ordering of the list of globs is maintained in the output.
@@ -60,7 +62,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.
 
 		// Filter matches to ones we haven't seen yet.
 		var includes []string
-		for _, match := range matches {
+		for i, match := range matches {
 			rel, err := filepath.Rel(b.BundleRootPath, match)
 			if err != nil {
 				return diag.FromErr(err)
@@ -69,9 +71,22 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.
 				continue
 			}
 			seen[rel] = true
+			if filepath.Ext(rel) != ".yaml" && filepath.Ext(rel) != ".yml" {
+				diags = diags.Append(diag.Diagnostic{
+					Severity:  diag.Error,
+					Summary:   "Files in the 'include' configuration section must be YAML files.",
+					Detail:    fmt.Sprintf("The file %s in the 'include' configuration section is not a YAML file, and only YAML files are supported. To include files to sync, specify them in the 'sync.include' configuration section instead.", rel),
+					Locations: b.Config.GetLocations(fmt.Sprintf("include[%d]", i)),
+				})
+				continue
+			}
 			includes = append(includes, rel)
 		}
 
+		if len(diags) > 0 {
+			return diags
+		}
+
 		// Add matches to list of mutators to return.
 		slices.Sort(includes)
 		files = append(files, includes...)

From 0487e816cc8c20272d12cf2f0ddde85a8a258c74 Mon Sep 17 00:00:00 2001
From: Ilya Kuznetsov <ilya.kuznetsov@databricks.com>
Date: Thu, 23 Jan 2025 15:35:33 +0100
Subject: [PATCH 138/247] Reading variables from file (#2171)

## Changes

New source of default values for variables - variable file
`.databricks/bundle/<target>/variable-overrides.json`

CLI tries to stat and read that file every time during variable
initialisation phase

<!-- Summary of your changes that are easy to understand -->

## Tests

Acceptance tests
---
 acceptance/bundle/variables/empty/output.txt  |  2 +-
 .../bundle/variables/env_overrides/output.txt |  2 +-
 .../complex_to_string/variable-overrides.json |  5 ++
 .../bundle/default/variable-overrides.json    |  7 ++
 .../invalid_json/variable-overrides.json      |  1 +
 .../string_to_complex/variable-overrides.json |  3 +
 .../bundle/with_value/variable-overrides.json |  3 +
 .../without_defaults/variable-overrides.json  |  4 +
 .../variable-overrides.json                   |  3 +
 .../bundle/variables/file-defaults/.gitignore |  1 +
 .../variables/file-defaults/databricks.yml    | 53 ++++++++++++
 .../bundle/variables/file-defaults/output.txt | 82 +++++++++++++++++++
 .../bundle/variables/file-defaults/script     | 30 +++++++
 .../bundle/variables/vanilla/output.txt       |  2 +-
 acceptance/script.prepare                     |  5 ++
 bundle/config/mutator/set_variables.go        | 69 +++++++++++++++-
 bundle/config/mutator/set_variables_test.go   | 14 ++--
 bundle/config/variable/variable.go            | 11 +--
 18 files changed, 278 insertions(+), 19 deletions(-)
 create mode 100644 acceptance/bundle/variables/file-defaults/.databricks/bundle/complex_to_string/variable-overrides.json
 create mode 100644 acceptance/bundle/variables/file-defaults/.databricks/bundle/default/variable-overrides.json
 create mode 100644 acceptance/bundle/variables/file-defaults/.databricks/bundle/invalid_json/variable-overrides.json
 create mode 100644 acceptance/bundle/variables/file-defaults/.databricks/bundle/string_to_complex/variable-overrides.json
 create mode 100644 acceptance/bundle/variables/file-defaults/.databricks/bundle/with_value/variable-overrides.json
 create mode 100644 acceptance/bundle/variables/file-defaults/.databricks/bundle/without_defaults/variable-overrides.json
 create mode 100644 acceptance/bundle/variables/file-defaults/.databricks/bundle/wrong_file_structure/variable-overrides.json
 create mode 100644 acceptance/bundle/variables/file-defaults/.gitignore
 create mode 100644 acceptance/bundle/variables/file-defaults/databricks.yml
 create mode 100644 acceptance/bundle/variables/file-defaults/output.txt
 create mode 100644 acceptance/bundle/variables/file-defaults/script

diff --git a/acceptance/bundle/variables/empty/output.txt b/acceptance/bundle/variables/empty/output.txt
index 261635920..8933443df 100644
--- a/acceptance/bundle/variables/empty/output.txt
+++ b/acceptance/bundle/variables/empty/output.txt
@@ -1,4 +1,4 @@
-Error: no value assigned to required variable a. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_a environment variable
+Error: no value assigned to required variable a. Assignment can be done using "--var", by setting the BUNDLE_VAR_a environment variable, or in .databricks/bundle/<target>/variable-overrides.json file
 
 Name: empty${var.a}
 Target: default
diff --git a/acceptance/bundle/variables/env_overrides/output.txt b/acceptance/bundle/variables/env_overrides/output.txt
index f42f82211..1ee9ef625 100644
--- a/acceptance/bundle/variables/env_overrides/output.txt
+++ b/acceptance/bundle/variables/env_overrides/output.txt
@@ -9,7 +9,7 @@
 "prod-a env-var-b"
 
 >>> errcode $CLI bundle validate -t env-missing-a-required-variable-assignment
-Error: no value assigned to required variable b. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_b environment variable
+Error: no value assigned to required variable b. Assignment can be done using "--var", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle/<target>/variable-overrides.json file
 
 Name: test bundle
 Target: env-missing-a-required-variable-assignment
diff --git a/acceptance/bundle/variables/file-defaults/.databricks/bundle/complex_to_string/variable-overrides.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/complex_to_string/variable-overrides.json
new file mode 100644
index 000000000..602567a68
--- /dev/null
+++ b/acceptance/bundle/variables/file-defaults/.databricks/bundle/complex_to_string/variable-overrides.json
@@ -0,0 +1,5 @@
+{
+  "cluster_key": {
+    "node_type_id": "Standard_DS3_v2"
+  }
+}
diff --git a/acceptance/bundle/variables/file-defaults/.databricks/bundle/default/variable-overrides.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/default/variable-overrides.json
new file mode 100644
index 000000000..3a865e120
--- /dev/null
+++ b/acceptance/bundle/variables/file-defaults/.databricks/bundle/default/variable-overrides.json
@@ -0,0 +1,7 @@
+{
+  "cluster": {
+    "node_type_id": "Standard_DS3_v2"
+  },
+  "cluster_key": "mlops_stacks-cluster",
+  "cluster_workers": 2
+}
diff --git a/acceptance/bundle/variables/file-defaults/.databricks/bundle/invalid_json/variable-overrides.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/invalid_json/variable-overrides.json
new file mode 100644
index 000000000..257cc5642
--- /dev/null
+++ b/acceptance/bundle/variables/file-defaults/.databricks/bundle/invalid_json/variable-overrides.json
@@ -0,0 +1 @@
+foo
diff --git a/acceptance/bundle/variables/file-defaults/.databricks/bundle/string_to_complex/variable-overrides.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/string_to_complex/variable-overrides.json
new file mode 100644
index 000000000..1ea719446
--- /dev/null
+++ b/acceptance/bundle/variables/file-defaults/.databricks/bundle/string_to_complex/variable-overrides.json
@@ -0,0 +1,3 @@
+{
+  "cluster": "mlops_stacks-cluster"
+}
diff --git a/acceptance/bundle/variables/file-defaults/.databricks/bundle/with_value/variable-overrides.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/with_value/variable-overrides.json
new file mode 100644
index 000000000..686d68548
--- /dev/null
+++ b/acceptance/bundle/variables/file-defaults/.databricks/bundle/with_value/variable-overrides.json
@@ -0,0 +1,3 @@
+{
+  "cluster_key": "mlops_stacks-cluster-from-file"
+}
diff --git a/acceptance/bundle/variables/file-defaults/.databricks/bundle/without_defaults/variable-overrides.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/without_defaults/variable-overrides.json
new file mode 100644
index 000000000..86166408e
--- /dev/null
+++ b/acceptance/bundle/variables/file-defaults/.databricks/bundle/without_defaults/variable-overrides.json
@@ -0,0 +1,4 @@
+{
+  "cluster_key": "mlops_stacks-cluster",
+  "cluster_workers": 2
+}
diff --git a/acceptance/bundle/variables/file-defaults/.databricks/bundle/wrong_file_structure/variable-overrides.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/wrong_file_structure/variable-overrides.json
new file mode 100644
index 000000000..de140ba36
--- /dev/null
+++ b/acceptance/bundle/variables/file-defaults/.databricks/bundle/wrong_file_structure/variable-overrides.json
@@ -0,0 +1,3 @@
+[
+  "foo"
+]
diff --git a/acceptance/bundle/variables/file-defaults/.gitignore b/acceptance/bundle/variables/file-defaults/.gitignore
new file mode 100644
index 000000000..bd1711fd1
--- /dev/null
+++ b/acceptance/bundle/variables/file-defaults/.gitignore
@@ -0,0 +1 @@
+!.databricks
diff --git a/acceptance/bundle/variables/file-defaults/databricks.yml b/acceptance/bundle/variables/file-defaults/databricks.yml
new file mode 100644
index 000000000..5838843e1
--- /dev/null
+++ b/acceptance/bundle/variables/file-defaults/databricks.yml
@@ -0,0 +1,53 @@
+bundle:
+  name: TestResolveVariablesFromFile
+
+variables:
+  cluster:
+    type: "complex"
+  cluster_key:
+  cluster_workers:
+
+resources:
+  jobs:
+    job1:
+      job_clusters:
+        - job_cluster_key: ${var.cluster_key}
+          new_cluster:
+            node_type_id: "${var.cluster.node_type_id}"
+            num_workers: ${var.cluster_workers}
+
+targets:
+  default:
+    default: true
+    variables:
+      cluster_workers: 1
+      cluster:
+        node_type_id: "default"
+      cluster_key: "default"
+
+  without_defaults:
+
+  complex_to_string:
+    variables:
+      cluster_workers: 1
+      cluster:
+        node_type_id: "default"
+      cluster_key: "default"
+
+  string_to_complex:
+    variables:
+      cluster_workers: 1
+      cluster:
+        node_type_id: "default"
+      cluster_key: "default"
+
+  wrong_file_structure:
+
+  invalid_json:
+
+  with_value:
+    variables:
+      cluster_workers: 1
+      cluster:
+        node_type_id: "default"
+      cluster_key: cluster_key_value
diff --git a/acceptance/bundle/variables/file-defaults/output.txt b/acceptance/bundle/variables/file-defaults/output.txt
new file mode 100644
index 000000000..73830aae3
--- /dev/null
+++ b/acceptance/bundle/variables/file-defaults/output.txt
@@ -0,0 +1,82 @@
+
+=== variable file
+>>> $CLI bundle validate -o json
+{
+  "job_cluster_key": "mlops_stacks-cluster",
+  "new_cluster": {
+    "node_type_id": "Standard_DS3_v2",
+    "num_workers": 2
+  }
+}
+
+=== variable file and variable flag
+>>> $CLI bundle validate -o json --var=cluster_key=mlops_stacks-cluster-overriden
+{
+  "job_cluster_key": "mlops_stacks-cluster-overriden",
+  "new_cluster": {
+    "node_type_id": "Standard_DS3_v2",
+    "num_workers": 2
+  }
+}
+
+=== variable file and environment variable
+>>> BUNDLE_VAR_cluster_key=mlops_stacks-cluster-overriden $CLI bundle validate -o json
+{
+  "job_cluster_key": "mlops_stacks-cluster-overriden",
+  "new_cluster": {
+    "node_type_id": "Standard_DS3_v2",
+    "num_workers": 2
+  }
+}
+
+=== variable has value in config file
+>>> $CLI bundle validate -o json --target with_value
+{
+  "job_cluster_key": "mlops_stacks-cluster-from-file",
+  "new_cluster": {
+    "node_type_id": "default",
+    "num_workers": 1
+  }
+}
+
+=== file has variable that is complex but default is string
+>>> errcode $CLI bundle validate -o json --target complex_to_string
+Error: variable cluster_key is not of type complex, but the value in the variable file is a complex type
+
+
+Exit code: 1
+{
+  "job_cluster_key": "${var.cluster_key}",
+  "new_cluster": {
+    "node_type_id": "${var.cluster.node_type_id}",
+    "num_workers": "${var.cluster_workers}"
+  }
+}
+
+=== file has variable that is string but default is complex
+>>> errcode $CLI bundle validate -o json --target string_to_complex
+Error: variable cluster is of type complex, but the value in the variable file is not a complex type
+
+
+Exit code: 1
+{
+  "job_cluster_key": "${var.cluster_key}",
+  "new_cluster": {
+    "node_type_id": "${var.cluster.node_type_id}",
+    "num_workers": "${var.cluster_workers}"
+  }
+}
+
+=== variable is required but it's not provided in the file
+>>> errcode $CLI bundle validate -o json --target without_defaults
+Error: no value assigned to required variable cluster. Assignment can be done using "--var", by setting the BUNDLE_VAR_cluster environment variable, or in .databricks/bundle/<target>/variable-overrides.json file
+
+
+Exit code: 1
+{
+  "job_cluster_key": "${var.cluster_key}",
+  "new_cluster": {
+    "node_type_id": "${var.cluster.node_type_id}",
+    "num_workers": "${var.cluster_workers}"
+  }
+}
diff --git a/acceptance/bundle/variables/file-defaults/script b/acceptance/bundle/variables/file-defaults/script
new file mode 100644
index 000000000..c5b208755
--- /dev/null
+++ b/acceptance/bundle/variables/file-defaults/script
@@ -0,0 +1,30 @@
+cluster_expr=".resources.jobs.job1.job_clusters[0]"
+
+# defaults from variable file, see .databricks/bundle/<target>/variable-overrides.json
+
+title "variable file"
+trace $CLI bundle validate -o json | jq $cluster_expr
+
+title "variable file and variable flag"
+trace $CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster-overriden" | jq $cluster_expr
+
+title "variable file and environment variable"
+trace BUNDLE_VAR_cluster_key=mlops_stacks-cluster-overriden $CLI bundle validate -o json | jq $cluster_expr
+
+title "variable has value in config file"
+trace $CLI bundle validate -o json --target with_value | jq $cluster_expr
+
+# title "file cannot be parsed"
+# trace errcode $CLI bundle validate -o json --target invalid_json | jq $cluster_expr
+
+# title "file has wrong structure"
+# trace errcode $CLI bundle validate -o json --target wrong_file_structure | jq $cluster_expr
+
+title "file has variable that is complex but default is string"
+trace errcode $CLI bundle validate -o json --target complex_to_string | jq $cluster_expr
+
+title "file has variable that is string but default is complex"
+trace errcode $CLI bundle validate -o json --target string_to_complex | jq $cluster_expr
+
+title "variable is required but it's not provided in the file"
+trace errcode $CLI bundle validate -o json --target without_defaults | jq $cluster_expr
diff --git a/acceptance/bundle/variables/vanilla/output.txt b/acceptance/bundle/variables/vanilla/output.txt
index 1d88bd060..e98882bb0 100644
--- a/acceptance/bundle/variables/vanilla/output.txt
+++ b/acceptance/bundle/variables/vanilla/output.txt
@@ -3,7 +3,7 @@
 "abc def"
 
 >>> errcode $CLI bundle validate
-Error: no value assigned to required variable b. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_b environment variable
+Error: no value assigned to required variable b. Assignment can be done using "--var", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle/<target>/variable-overrides.json file
 
 Name: ${var.a} ${var.b}
 Target: default
diff --git a/acceptance/script.prepare b/acceptance/script.prepare
index 5900016d7..0567e433a 100644
--- a/acceptance/script.prepare
+++ b/acceptance/script.prepare
@@ -40,3 +40,8 @@ git-repo-init() {
     git add databricks.yml
     git commit -qm 'Add databricks.yml'
 }
+
+title() {
+    local label="$1"
+    printf "\n=== %s" "$label"
+}
diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go
index 9e9f2dcfe..ac2f660a9 100644
--- a/bundle/config/mutator/set_variables.go
+++ b/bundle/config/mutator/set_variables.go
@@ -3,11 +3,14 @@ package mutator
 import (
 	"context"
 	"fmt"
+	"os"
+	"path/filepath"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config/variable"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/jsonloader"
 	"github.com/databricks/cli/libs/env"
 )
 
@@ -23,7 +26,11 @@ func (m *setVariables) Name() string {
 	return "SetVariables"
 }
 
-func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, name string) (dyn.Value, error) {
+func getDefaultVariableFilePath(target string) string {
+	return ".databricks/bundle/" + target + "/variable-overrides.json"
+}
+
+func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, name string, fileDefault dyn.Value) (dyn.Value, error) {
 	// case: variable already has value initialized, so skip
 	if variable.HasValue() {
 		return v, nil
@@ -49,6 +56,26 @@ func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable,
 		return v, nil
 	}
 
+	// case: Set the variable to the default value from the variable file
+	if fileDefault.Kind() != dyn.KindInvalid && fileDefault.Kind() != dyn.KindNil {
+		hasComplexType := variable.IsComplex()
+		hasComplexValue := fileDefault.Kind() == dyn.KindMap || fileDefault.Kind() == dyn.KindSequence
+
+		if hasComplexType && !hasComplexValue {
+			return dyn.InvalidValue, fmt.Errorf(`variable %s is of type complex, but the value in the variable file is not a complex type`, name)
+		}
+		if !hasComplexType && hasComplexValue {
+			return dyn.InvalidValue, fmt.Errorf(`variable %s is not of type complex, but the value in the variable file is a complex type`, name)
+		}
+
+		v, err := dyn.Set(v, "value", fileDefault)
+		if err != nil {
+			return dyn.InvalidValue, fmt.Errorf(`failed to assign default value from variable file to variable %s with error: %v`, name, err)
+		}
+
+		return v, nil
+	}
+
 	// case: Set the variable to its default value
 	if variable.HasDefault() {
 		vDefault, err := dyn.Get(v, "default")
@@ -64,10 +91,43 @@ func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable,
 	}
 
 	// We should have had a value to set for the variable at this point.
-	return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" flag or by setting the %s environment variable`, name, bundleVarPrefix+name)
+	return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done using "--var", by setting the %s environment variable, or in %s file`, name, bundleVarPrefix+name, getDefaultVariableFilePath("<target>"))
+}
+
+func readVariablesFromFile(b *bundle.Bundle) (dyn.Value, diag.Diagnostics) {
+	var diags diag.Diagnostics
+
+	filePath := filepath.Join(b.BundleRootPath, getDefaultVariableFilePath(b.Config.Bundle.Target))
+	if _, err := os.Stat(filePath); err != nil {
+		return dyn.InvalidValue, nil
+	}
+
+	f, err := os.ReadFile(filePath)
+	if err != nil {
+		return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to read variables file: %w", err))
+	}
+
+	val, err := jsonloader.LoadJSON(f, filePath)
+	if err != nil {
+		return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to parse variables file %s: %w", filePath, err))
+	}
+
+	if val.Kind() != dyn.KindMap {
+		return dyn.InvalidValue, diags.Append(diag.Diagnostic{
+			Severity: diag.Error,
+			Summary:  fmt.Sprintf("failed to parse variables file %s: invalid format", filePath),
+			Detail:   "Variables file must be a JSON object with the following format:\n{\"var1\": \"value1\", \"var2\": \"value2\"}",
+		})
+	}
+
+	return val, nil
 }
 
 func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	defaults, diags := readVariablesFromFile(b)
+	if diags.HasError() {
+		return diags
+	}
 	err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
 		return dyn.Map(v, "variables", dyn.Foreach(func(p dyn.Path, variable dyn.Value) (dyn.Value, error) {
 			name := p[1].Key()
@@ -76,9 +136,10 @@ func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 				return dyn.InvalidValue, fmt.Errorf(`variable "%s" is not defined`, name)
 			}
 
-			return setVariable(ctx, variable, v, name)
+			fileDefault, _ := dyn.Get(defaults, name)
+			return setVariable(ctx, variable, v, name, fileDefault)
 		}))
 	})
 
-	return diag.FromErr(err)
+	return diags.Extend(diag.FromErr(err))
 }
diff --git a/bundle/config/mutator/set_variables_test.go b/bundle/config/mutator/set_variables_test.go
index 07a5c8214..d904d5be3 100644
--- a/bundle/config/mutator/set_variables_test.go
+++ b/bundle/config/mutator/set_variables_test.go
@@ -25,7 +25,7 @@ func TestSetVariableFromProcessEnvVar(t *testing.T) {
 	v, err := convert.FromTyped(variable, dyn.NilValue)
 	require.NoError(t, err)
 
-	v, err = setVariable(context.Background(), v, &variable, "foo")
+	v, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue)
 	require.NoError(t, err)
 
 	err = convert.ToTyped(&variable, v)
@@ -43,7 +43,7 @@ func TestSetVariableUsingDefaultValue(t *testing.T) {
 	v, err := convert.FromTyped(variable, dyn.NilValue)
 	require.NoError(t, err)
 
-	v, err = setVariable(context.Background(), v, &variable, "foo")
+	v, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue)
 	require.NoError(t, err)
 
 	err = convert.ToTyped(&variable, v)
@@ -65,7 +65,7 @@ func TestSetVariableWhenAlreadyAValueIsAssigned(t *testing.T) {
 	v, err := convert.FromTyped(variable, dyn.NilValue)
 	require.NoError(t, err)
 
-	v, err = setVariable(context.Background(), v, &variable, "foo")
+	v, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue)
 	require.NoError(t, err)
 
 	err = convert.ToTyped(&variable, v)
@@ -90,7 +90,7 @@ func TestSetVariableEnvVarValueDoesNotOverridePresetValue(t *testing.T) {
 	v, err := convert.FromTyped(variable, dyn.NilValue)
 	require.NoError(t, err)
 
-	v, err = setVariable(context.Background(), v, &variable, "foo")
+	v, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue)
 	require.NoError(t, err)
 
 	err = convert.ToTyped(&variable, v)
@@ -107,8 +107,8 @@ func TestSetVariablesErrorsIfAValueCouldNotBeResolved(t *testing.T) {
 	v, err := convert.FromTyped(variable, dyn.NilValue)
 	require.NoError(t, err)
 
-	_, err = setVariable(context.Background(), v, &variable, "foo")
-	assert.ErrorContains(t, err, "no value assigned to required variable foo. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_foo environment variable")
+	_, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue)
+	assert.ErrorContains(t, err, "no value assigned to required variable foo. Assignment can be done using \"--var\", by setting the BUNDLE_VAR_foo environment variable, or in .databricks/bundle/<target>/variable-overrides.json file")
 }
 
 func TestSetVariablesMutator(t *testing.T) {
@@ -157,6 +157,6 @@ func TestSetComplexVariablesViaEnvVariablesIsNotAllowed(t *testing.T) {
 	v, err := convert.FromTyped(variable, dyn.NilValue)
 	require.NoError(t, err)
 
-	_, err = setVariable(context.Background(), v, &variable, "foo")
+	_, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue)
 	assert.ErrorContains(t, err, "setting via environment variables (BUNDLE_VAR_foo) is not supported for complex variable foo")
 }
diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go
index 95a68cfeb..d7f1cdede 100644
--- a/bundle/config/variable/variable.go
+++ b/bundle/config/variable/variable.go
@@ -36,11 +36,12 @@ type Variable struct {
 	// This field stores the resolved value for the variable. The variable are
 	// resolved in the following priority order (from highest to lowest)
 	//
-	// 1. Command line flag. For example: `--var="foo=bar"`
-	// 2. Target variable. eg: BUNDLE_VAR_foo=bar
-	// 3. Default value as defined in the applicable environments block
-	// 4. Default value defined in variable definition
-	// 5. Throw error, since if no default value is defined, then the variable
+	// 1. Command line flag `--var="foo=bar"`
+	// 2. Environment variable. eg: BUNDLE_VAR_foo=bar
+	// 3. Load defaults from .databricks/bundle/<target>/variable-overrides.json
+	// 4. Default value as defined in the applicable targets block
+	// 5. Default value defined in variable definition
+	// 6. Throw error, since if no default value is defined, then the variable
 	//    is required
 	Value VariableValue `json:"value,omitempty" bundle:"readonly"`
 

From d784147e994f71ea7b4e30a02daea66e73baea10 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Thu, 23 Jan 2025 16:54:55 +0100
Subject: [PATCH 139/247] [Release] Release v0.239.1 (#2218)

CLI:
* Added text output templates for apps list and list-deployments
([#2175](https://github.com/databricks/cli/pull/2175)).
* Fix duplicate "apps" entry in help output
([#2191](https://github.com/databricks/cli/pull/2191)).

Bundles:
* Allow yaml-anchors in schema
([#2200](https://github.com/databricks/cli/pull/2200)).
* Show an error when non-yaml files used in include section
([#2201](https://github.com/databricks/cli/pull/2201)).
* Set WorktreeRoot to sync root outside git repo
([#2197](https://github.com/databricks/cli/pull/2197)).
* fix: Detailed message for using source-linked deployment with
file_path specified
([#2119](https://github.com/databricks/cli/pull/2119)).
* Allow using variables in enum fields
([#2199](https://github.com/databricks/cli/pull/2199)).
* Add experimental-jobs-as-code template
([#2177](https://github.com/databricks/cli/pull/2177)).
* Reading variables from file
([#2171](https://github.com/databricks/cli/pull/2171)).
* Fixed an apps message order and added output test
([#2174](https://github.com/databricks/cli/pull/2174)).
* Default to forward slash-separated paths for path translation
([#2145](https://github.com/databricks/cli/pull/2145)).
* Include a materialized copy of built-in templates
([#2146](https://github.com/databricks/cli/pull/2146)).
---
 CHANGELOG.md | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 53392e5db..255bfb0a8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,25 @@
 # Version changelog
 
+## [Release] Release v0.239.1
+
+CLI:
+ * Added text output templates for apps list and list-deployments ([#2175](https://github.com/databricks/cli/pull/2175)).
+ * Fix duplicate "apps" entry in help output ([#2191](https://github.com/databricks/cli/pull/2191)).
+
+Bundles:
+ * Allow yaml-anchors in schema ([#2200](https://github.com/databricks/cli/pull/2200)).
+ * Show an error when non-yaml files used in include section ([#2201](https://github.com/databricks/cli/pull/2201)).
+ * Set WorktreeRoot to sync root outside git repo ([#2197](https://github.com/databricks/cli/pull/2197)).
+ * fix: Detailed message for using source-linked deployment with file_path specified ([#2119](https://github.com/databricks/cli/pull/2119)).
+ * Allow using variables in enum fields ([#2199](https://github.com/databricks/cli/pull/2199)).
+ * Add experimental-jobs-as-code template ([#2177](https://github.com/databricks/cli/pull/2177)).
+ * Reading variables from file ([#2171](https://github.com/databricks/cli/pull/2171)).
+ * Fixed an apps message order and added output test ([#2174](https://github.com/databricks/cli/pull/2174)).
+ * Default to forward slash-separated paths for path translation ([#2145](https://github.com/databricks/cli/pull/2145)).
+ * Include a materialized copy of built-in templates ([#2146](https://github.com/databricks/cli/pull/2146)).
+
+
+
 ## [Release] Release v0.239.0
 
 ### New feature announcement

From d6d9b994d46bdd11bdc17f215a4138558b626457 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 24 Jan 2025 10:47:12 +0100
Subject: [PATCH 140/247] acc: only print non-zero exit codes in errcode
 function (#2222)

Reduce noise in the output and matches how "Exit code" is handled for
the whole script.
---
 acceptance/bundle/paths/fallback/output.txt       | 2 --
 acceptance/bundle/paths/nominal/output.txt        | 2 --
 acceptance/bundle/variables/arg-repeat/output.txt | 2 --
 acceptance/script.prepare                         | 4 +++-
 4 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/acceptance/bundle/paths/fallback/output.txt b/acceptance/bundle/paths/fallback/output.txt
index f694610d2..63121f3d7 100644
--- a/acceptance/bundle/paths/fallback/output.txt
+++ b/acceptance/bundle/paths/fallback/output.txt
@@ -1,8 +1,6 @@
 
 >>> $CLI bundle validate -t development -o json
 
-Exit code: 0
-
 >>> $CLI bundle validate -t error
 Error: notebook this value is overridden not found. Local notebook references are expected
 to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb]
diff --git a/acceptance/bundle/paths/nominal/output.txt b/acceptance/bundle/paths/nominal/output.txt
index 189170335..1badcdec6 100644
--- a/acceptance/bundle/paths/nominal/output.txt
+++ b/acceptance/bundle/paths/nominal/output.txt
@@ -1,8 +1,6 @@
 
 >>> $CLI bundle validate -t development -o json
 
-Exit code: 0
-
 >>> $CLI bundle validate -t error
 Error: notebook this value is overridden not found. Local notebook references are expected
 to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb]
diff --git a/acceptance/bundle/variables/arg-repeat/output.txt b/acceptance/bundle/variables/arg-repeat/output.txt
index 48bd2033f..2f9de1a3c 100644
--- a/acceptance/bundle/variables/arg-repeat/output.txt
+++ b/acceptance/bundle/variables/arg-repeat/output.txt
@@ -1,7 +1,5 @@
 
 >>> errcode $CLI bundle validate --var a=one -o json
-
-Exit code: 0
 {
   "a": {
     "default": "hello",
diff --git a/acceptance/script.prepare b/acceptance/script.prepare
index 0567e433a..87910654d 100644
--- a/acceptance/script.prepare
+++ b/acceptance/script.prepare
@@ -6,7 +6,9 @@ errcode() {
     local exit_code=$?
     # Re-enable 'set -e' if it was previously set
     set -e
-    >&2 printf "\nExit code: $exit_code\n"
+    if [ $exit_code -ne 0 ]; then
+        >&2 printf "\nExit code: $exit_code\n"
+    fi
 }
 
 trace() {

From b4ed23510490bcc16e15990c210598341d4657a6 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 24 Jan 2025 11:18:44 +0100
Subject: [PATCH 141/247] Include EvalSymlinks in SetPath and use SetPath on
 all paths (#2219)

## Changes
When adding path, a few things should take care of:
- symlink expansion
- forward/backward slashes, so that tests could do sed 's/\\\\/\//g' to
make it pass on Windows (see
acceptance/bundle/syncroot/dotdot-git/script)

SetPath() function takes care of both.

This PR uses SetPath() on all paths consistently.

## Tests
Existing tests.
---
 acceptance/acceptance_test.go |  9 ++-------
 libs/testdiff/replacement.go  | 16 ++++++++++++++--
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 96c1f651c..0e7877dcf 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -93,13 +93,13 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	}
 
 	t.Setenv("CLI", execPath)
-	repls.Set(execPath, "$CLI")
+	repls.SetPath(execPath, "$CLI")
 
 	// Make helper scripts available
 	t.Setenv("PATH", fmt.Sprintf("%s%c%s", filepath.Join(cwd, "bin"), os.PathListSeparator, os.Getenv("PATH")))
 
 	tempHomeDir := t.TempDir()
-	repls.Set(tempHomeDir, "$TMPHOME")
+	repls.SetPath(tempHomeDir, "$TMPHOME")
 	t.Logf("$TMPHOME=%v", tempHomeDir)
 
 	// Prevent CLI from downloading terraform in each test:
@@ -187,11 +187,6 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		tmpDir = t.TempDir()
 	}
 
-	// Converts C:\Users\DENIS~1.BIL -> C:\Users\denis.bilenko
-	tmpDirEvalled, err1 := filepath.EvalSymlinks(tmpDir)
-	if err1 == nil && tmpDirEvalled != tmpDir {
-		repls.SetPathWithParents(tmpDirEvalled, "$TMPDIR")
-	}
 	repls.SetPathWithParents(tmpDir, "$TMPDIR")
 
 	scriptContents := readMergedScriptContents(t, dir)
diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go
index ca76b159c..865192662 100644
--- a/libs/testdiff/replacement.go
+++ b/libs/testdiff/replacement.go
@@ -94,6 +94,18 @@ func trimQuotes(s string) string {
 }
 
 func (r *ReplacementsContext) SetPath(old, new string) {
+	if old != "" && old != "." {
+		// Converts C:\Users\DENIS~1.BIL -> C:\Users\denis.bilenko
+		oldEvalled, err1 := filepath.EvalSymlinks(old)
+		if err1 == nil && oldEvalled != old {
+			r.SetPathNoEval(oldEvalled, new)
+		}
+	}
+
+	r.SetPathNoEval(old, new)
+}
+
+func (r *ReplacementsContext) SetPathNoEval(old, new string) {
 	r.Set(old, new)
 
 	if runtime.GOOS != "windows" {
@@ -133,7 +145,7 @@ func PrepareReplacementsWorkspaceClient(t testutil.TestingT, r *ReplacementsCont
 	r.Set(w.Config.Token, "$DATABRICKS_TOKEN")
 	r.Set(w.Config.Username, "$DATABRICKS_USERNAME")
 	r.Set(w.Config.Password, "$DATABRICKS_PASSWORD")
-	r.Set(w.Config.Profile, "$DATABRICKS_CONFIG_PROFILE")
+	r.SetPath(w.Config.Profile, "$DATABRICKS_CONFIG_PROFILE")
 	r.Set(w.Config.ConfigFile, "$DATABRICKS_CONFIG_FILE")
 	r.Set(w.Config.GoogleServiceAccount, "$DATABRICKS_GOOGLE_SERVICE_ACCOUNT")
 	r.Set(w.Config.GoogleCredentials, "$GOOGLE_CREDENTIALS")
@@ -147,7 +159,7 @@ func PrepareReplacementsWorkspaceClient(t testutil.TestingT, r *ReplacementsCont
 	r.Set(w.Config.AzureEnvironment, "$ARM_ENVIRONMENT")
 	r.Set(w.Config.ClientID, "$DATABRICKS_CLIENT_ID")
 	r.Set(w.Config.ClientSecret, "$DATABRICKS_CLIENT_SECRET")
-	r.Set(w.Config.DatabricksCliPath, "$DATABRICKS_CLI_PATH")
+	r.SetPath(w.Config.DatabricksCliPath, "$DATABRICKS_CLI_PATH")
 	// This is set to words like "path" that happen too frequently
 	// r.Set(w.Config.AuthType, "$DATABRICKS_AUTH_TYPE")
 }

From a47a058506d874019887baea1006b587f47cbfdb Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Fri, 24 Jan 2025 16:35:00 +0530
Subject: [PATCH 142/247] Limit test server to only accept GET on read
 endpoints (#2225)

## Changes
Now the test server will only match GET queries for these endpoints

## Tests
Existing tests.
---
 acceptance/server_test.go | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/acceptance/server_test.go b/acceptance/server_test.go
index 0d10fbea1..eb8cbb24a 100644
--- a/acceptance/server_test.go
+++ b/acceptance/server_test.go
@@ -68,7 +68,7 @@ func StartServer(t *testing.T) *TestServer {
 }
 
 func AddHandlers(server *TestServer) {
-	server.Handle("/api/2.0/policies/clusters/list", func(r *http.Request) (any, error) {
+	server.Handle("GET /api/2.0/policies/clusters/list", func(r *http.Request) (any, error) {
 		return compute.ListPoliciesResponse{
 			Policies: []compute.Policy{
 				{
@@ -83,7 +83,7 @@ func AddHandlers(server *TestServer) {
 		}, nil
 	})
 
-	server.Handle("/api/2.0/instance-pools/list", func(r *http.Request) (any, error) {
+	server.Handle("GET /api/2.0/instance-pools/list", func(r *http.Request) (any, error) {
 		return compute.ListInstancePools{
 			InstancePools: []compute.InstancePoolAndStats{
 				{
@@ -94,7 +94,7 @@ func AddHandlers(server *TestServer) {
 		}, nil
 	})
 
-	server.Handle("/api/2.1/clusters/list", func(r *http.Request) (any, error) {
+	server.Handle("GET /api/2.1/clusters/list", func(r *http.Request) (any, error) {
 		return compute.ListClustersResponse{
 			Clusters: []compute.ClusterDetails{
 				{
@@ -109,13 +109,13 @@ func AddHandlers(server *TestServer) {
 		}, nil
 	})
 
-	server.Handle("/api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, error) {
+	server.Handle("GET /api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, error) {
 		return iam.User{
 			UserName: "tester@databricks.com",
 		}, nil
 	})
 
-	server.Handle("/api/2.0/workspace/get-status", func(r *http.Request) (any, error) {
+	server.Handle("GET /api/2.0/workspace/get-status", func(r *http.Request) (any, error) {
 		return workspace.ObjectInfo{
 			ObjectId:   1001,
 			ObjectType: "DIRECTORY",
@@ -124,13 +124,13 @@ func AddHandlers(server *TestServer) {
 		}, nil
 	})
 
-	server.Handle("/api/2.1/unity-catalog/current-metastore-assignment", func(r *http.Request) (any, error) {
+	server.Handle("GET /api/2.1/unity-catalog/current-metastore-assignment", func(r *http.Request) (any, error) {
 		return catalog.MetastoreAssignment{
 			DefaultCatalogName: "main",
 		}, nil
 	})
 
-	server.Handle("/api/2.0/permissions/directories/1001", func(r *http.Request) (any, error) {
+	server.Handle("GET /api/2.0/permissions/directories/1001", func(r *http.Request) (any, error) {
 		return workspace.WorkspaceObjectPermissions{
 			ObjectId:   "1001",
 			ObjectType: "DIRECTORY",

From 959e43e556b2fc775feaf5d519000afdad17a815 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 24 Jan 2025 15:28:23 +0100
Subject: [PATCH 143/247] acc: Support per-test configuration; GOOS option to
 disable OS (#2227)

## Changes
- Acceptance tests load test.toml to configure test behaviour.
- If file is not found in the test directory, parents are searched,
until the test root.
- Currently there is one option: runtime.GOOS to switch off tests per
OS.

## Tests
Using it in https://github.com/databricks/cli/pull/2223 to disable test
on Windows that cannot be run there.
---
 NOTICE                        |  4 ++
 acceptance/acceptance_test.go |  7 +++
 acceptance/config_test.go     | 99 +++++++++++++++++++++++++++++++++++
 acceptance/test.toml          |  2 +
 go.mod                        |  1 +
 go.sum                        |  2 +
 6 files changed, 115 insertions(+)
 create mode 100644 acceptance/config_test.go
 create mode 100644 acceptance/test.toml

diff --git a/NOTICE b/NOTICE
index f6b59e0b0..ed22084cf 100644
--- a/NOTICE
+++ b/NOTICE
@@ -105,3 +105,7 @@ License - https://github.com/wI2L/jsondiff/blob/master/LICENSE
 https://github.com/hexops/gotextdiff
 Copyright (c) 2009 The Go Authors. All rights reserved.
 License - https://github.com/hexops/gotextdiff/blob/main/LICENSE
+
+https://github.com/BurntSushi/toml
+Copyright (c) 2013 TOML authors
+https://github.com/BurntSushi/toml/blob/master/COPYING
diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 0e7877dcf..a1c41c5e6 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -175,6 +175,13 @@ func getTests(t *testing.T) []string {
 }
 
 func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsContext) {
+	config, configPath := LoadConfig(t, dir)
+
+	isEnabled, isPresent := config.GOOS[runtime.GOOS]
+	if isPresent && !isEnabled {
+		t.Skipf("Disabled via GOOS.%s setting in %s", runtime.GOOS, configPath)
+	}
+
 	var tmpDir string
 	var err error
 	if KeepTmp {
diff --git a/acceptance/config_test.go b/acceptance/config_test.go
new file mode 100644
index 000000000..49dce06ba
--- /dev/null
+++ b/acceptance/config_test.go
@@ -0,0 +1,99 @@
+package acceptance_test
+
+import (
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+
+	"github.com/BurntSushi/toml"
+	"github.com/stretchr/testify/require"
+)
+
+const configFilename = "test.toml"
+
+var (
+	configCache map[string]TestConfig
+	configMutex sync.Mutex
+)
+
+type TestConfig struct {
+	// Place to describe what's wrong with this test. Does not affect how the test is run.
+	Badness string
+
+	// Which OSes the test is enabled on. Each string is compared against runtime.GOOS.
+	// If absent, default to true.
+	GOOS map[string]bool
+}
+
+// FindConfig finds the closest config file.
+func FindConfig(t *testing.T, dir string) (string, bool) {
+	shared := false
+	for {
+		path := filepath.Join(dir, configFilename)
+		_, err := os.Stat(path)
+
+		if err == nil {
+			return path, shared
+		}
+
+		shared = true
+
+		if dir == "" || dir == "." {
+			break
+		}
+
+		if os.IsNotExist(err) {
+			dir = filepath.Dir(dir)
+			continue
+		}
+
+		t.Fatalf("Error while reading %s: %s", path, err)
+	}
+
+	t.Fatal("Config not found: " + configFilename)
+	return "", shared
+}
+
+// LoadConfig loads the config file. Non-leaf configs are cached.
+func LoadConfig(t *testing.T, dir string) (TestConfig, string) {
+	path, leafConfig := FindConfig(t, dir)
+
+	if leafConfig {
+		return DoLoadConfig(t, path), path
+	}
+
+	configMutex.Lock()
+	defer configMutex.Unlock()
+
+	if configCache == nil {
+		configCache = make(map[string]TestConfig)
+	}
+
+	result, ok := configCache[path]
+	if ok {
+		return result, path
+	}
+
+	result = DoLoadConfig(t, path)
+	configCache[path] = result
+	return result, path
+}
+
+func DoLoadConfig(t *testing.T, path string) TestConfig {
+	bytes, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("failed to read config: %s", err)
+	}
+
+	var config TestConfig
+	meta, err := toml.Decode(string(bytes), &config)
+	require.NoError(t, err)
+
+	keys := meta.Undecoded()
+	if len(keys) > 0 {
+		t.Fatalf("Undecoded keys in %s: %#v", path, keys)
+	}
+
+	return config
+}
diff --git a/acceptance/test.toml b/acceptance/test.toml
new file mode 100644
index 000000000..eee94d0ea
--- /dev/null
+++ b/acceptance/test.toml
@@ -0,0 +1,2 @@
+# If test directory nor any of its parents do not have test.toml then this file serves as fallback configuration.
+# The configurations are not merged across parents; the closest one is used fully.
diff --git a/go.mod b/go.mod
index 0ef800d7b..930963f89 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,7 @@ go 1.23
 toolchain go1.23.4
 
 require (
+	github.com/BurntSushi/toml v1.4.0 // MIT
 	github.com/Masterminds/semver/v3 v3.3.1 // MIT
 	github.com/briandowns/spinner v1.23.1 // Apache 2.0
 	github.com/databricks/databricks-sdk-go v0.55.0 // Apache 2.0
diff --git a/go.sum b/go.sum
index b1364cb26..d025b3947 100644
--- a/go.sum
+++ b/go.sum
@@ -8,6 +8,8 @@ cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1h
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
+github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4=
 github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=

From f65508690d92301e0f6e27ce76a46d28780272ea Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Fri, 24 Jan 2025 16:33:54 +0100
Subject: [PATCH 144/247] Update publish-winget action to use Komac directly
 (#2228)

## Changes

For the most recent release, I had to re-run the "publish-winget" action
a couple of times before it passed. The underlying issue that causes the
failure should be solved by the latest version of the action, but upon
inspection of the latest version, I found that it always installs the
latest version of [Komac](https://github.com/russellbanks/Komac). To
both fix the issue and lock this down further, I updated our action to
call Komac directly instead of relying on a separate action to do this
for us.

## Tests

Successful run in
https://github.com/databricks/cli/actions/runs/12951529979.
---
 .github/workflows/publish-winget.yml | 68 +++++++++++++++++++++++-----
 1 file changed, 57 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/publish-winget.yml b/.github/workflows/publish-winget.yml
index eb9a72eda..cbd24856b 100644
--- a/.github/workflows/publish-winget.yml
+++ b/.github/workflows/publish-winget.yml
@@ -10,19 +10,65 @@ on:
 jobs:
   publish-to-winget-pkgs:
     runs-on:
-      group: databricks-protected-runner-group
-      labels: windows-server-latest
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
 
     environment: release
 
     steps:
-      - uses: vedantmgoyal2009/winget-releaser@93fd8b606a1672ec3e5c6c3bb19426be68d1a8b0 # v2
-        with:
-          identifier: Databricks.DatabricksCLI
-          installers-regex: 'windows_.*-signed\.zip$' # Only signed Windows releases
-          token: ${{ secrets.ENG_DEV_ECOSYSTEM_BOT_TOKEN }}
-          fork-user: eng-dev-ecosystem-bot
+      - name: Checkout repository and submodules
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
-          # Use the tag from the input, or the ref name if the input is not provided.
-          # The ref name is equal to the tag name when this workflow is triggered by the "sign-cli" command.
-          release-tag: ${{ inputs.tag || github.ref_name }}
+        # When updating the version of komac, make sure to update the checksum in the next step.
+        # Find both at https://github.com/russellbanks/Komac/releases.
+      - name: Download komac binary
+        run: |
+          curl -s -L -o $RUNNER_TEMP/komac-2.9.0-x86_64-unknown-linux-gnu.tar.gz https://github.com/russellbanks/Komac/releases/download/v2.9.0/komac-2.9.0-x86_64-unknown-linux-gnu.tar.gz
+
+      - name: Verify komac binary
+        run: |
+          echo "d07a12831ad5418fee715488542a98ce3c0e591d05c850dd149fe78432be8c4c  $RUNNER_TEMP/komac-2.9.0-x86_64-unknown-linux-gnu.tar.gz" | sha256sum -c -
+
+      - name: Untar komac binary to temporary path
+        run: |
+          mkdir -p $RUNNER_TEMP/komac
+          tar -xzf $RUNNER_TEMP/komac-2.9.0-x86_64-unknown-linux-gnu.tar.gz -C $RUNNER_TEMP/komac
+
+      - name: Add komac to PATH
+        run: echo "$RUNNER_TEMP/komac" >> $GITHUB_PATH
+
+      - name: Confirm komac version
+        run: komac --version
+
+        # Use the tag from the input, or the ref name if the input is not provided.
+        # The ref name is equal to the tag name when this workflow is triggered by the "sign-cli" command.
+      - name: Strip "v" prefix from version
+        id: strip_version
+        run: echo "version=$(echo ${{ inputs.tag || github.ref_name }} | sed 's/^v//')" >> "$GITHUB_OUTPUT"
+
+      - name: Get URLs of signed Windows binaries
+        id: get_windows_urls
+        run: |
+          urls=$(
+            gh api https://api.github.com/repos/databricks/cli/releases/tags/${{ inputs.tag || github.ref_name }} | \
+            jq -r .assets[].browser_download_url | \
+            grep -E '_windows_.*-signed\.zip$' | \
+            tr '\n' ' '
+          )
+          if [ -z "$urls" ]; then
+            echo "No signed Windows binaries found" >&2
+            exit 1
+          fi
+          echo "urls=$urls" >> "$GITHUB_OUTPUT"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Publish to Winget
+        run: |
+          komac update Databricks.DatabricksCLI \
+            --version ${{ steps.strip_version.outputs.version }} \
+            --submit \
+            --urls ${{ steps.get_windows_urls.outputs.urls }} \
+        env:
+          KOMAC_FORK_OWNER: eng-dev-ecosystem-bot
+          GITHUB_TOKEN: ${{ secrets.ENG_DEV_ECOSYSTEM_BOT_TOKEN }}

From 468660dc45bd1deac4d37fb914d4a6224aa1a27e Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 24 Jan 2025 16:53:06 +0100
Subject: [PATCH 145/247] Add an acc test covering failures when reading .git
 (#2223)

## Changes
- New test covering failures in reading .git. One case results in error,
some result in warning (not shown).
- New helper withdir runs commands in a subdirectory.

## Tests
New acceptance test.
---
 .../bundle/git-permerror/databricks.yml       |  2 +
 acceptance/bundle/git-permerror/output.txt    | 78 +++++++++++++++++++
 acceptance/bundle/git-permerror/script        | 25 ++++++
 acceptance/bundle/git-permerror/test.toml     |  5 ++
 acceptance/script.prepare                     | 11 +++
 5 files changed, 121 insertions(+)
 create mode 100644 acceptance/bundle/git-permerror/databricks.yml
 create mode 100644 acceptance/bundle/git-permerror/output.txt
 create mode 100644 acceptance/bundle/git-permerror/script
 create mode 100644 acceptance/bundle/git-permerror/test.toml

diff --git a/acceptance/bundle/git-permerror/databricks.yml b/acceptance/bundle/git-permerror/databricks.yml
new file mode 100644
index 000000000..83e0acda8
--- /dev/null
+++ b/acceptance/bundle/git-permerror/databricks.yml
@@ -0,0 +1,2 @@
+bundle:
+  name: git-permerror
diff --git a/acceptance/bundle/git-permerror/output.txt b/acceptance/bundle/git-permerror/output.txt
new file mode 100644
index 000000000..2b52134ab
--- /dev/null
+++ b/acceptance/bundle/git-permerror/output.txt
@@ -0,0 +1,78 @@
+=== No permission to access .git. Badness: inferred flag is set to true even though we did not infer branch. bundle_root_path is not correct in subdir case.
+
+>>> chmod 000 .git
+
+>>> $CLI bundle validate
+Error: unable to load repository specific gitconfig: open config: permission denied
+
+Name: git-permerror
+Target: default
+Workspace:
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/git-permerror/default
+
+Found 1 error
+
+Exit code: 1
+
+>>> $CLI bundle validate -o json
+Error: unable to load repository specific gitconfig: open config: permission denied
+
+
+Exit code: 1
+{
+  "bundle_root_path": ".",
+  "inferred": true
+}
+
+>>> withdir subdir/a/b $CLI bundle validate -o json
+Error: unable to load repository specific gitconfig: open config: permission denied
+
+
+Exit code: 1
+{
+  "bundle_root_path": ".",
+  "inferred": true
+}
+
+
+=== No permissions to read .git/HEAD. Badness: warning is not shown. inferred is incorrectly set to true. bundle_root_path is not correct in subdir case.
+
+>>> chmod 000 .git/HEAD
+
+>>> $CLI bundle validate -o json
+{
+  "bundle_root_path": ".",
+  "inferred": true
+}
+
+>>> withdir subdir/a/b $CLI bundle validate -o json
+{
+  "bundle_root_path": ".",
+  "inferred": true
+}
+
+
+=== No permissions to read .git/config. Badness: inferred is incorretly set to true. bundle_root_path is not correct is subdir case.
+
+>>> chmod 000 .git/config
+
+>>> $CLI bundle validate -o json
+Error: unable to load repository specific gitconfig: open config: permission denied
+
+
+Exit code: 1
+{
+  "bundle_root_path": ".",
+  "inferred": true
+}
+
+>>> withdir subdir/a/b $CLI bundle validate -o json
+Error: unable to load repository specific gitconfig: open config: permission denied
+
+
+Exit code: 1
+{
+  "bundle_root_path": ".",
+  "inferred": true
+}
diff --git a/acceptance/bundle/git-permerror/script b/acceptance/bundle/git-permerror/script
new file mode 100644
index 000000000..782cbf5bc
--- /dev/null
+++ b/acceptance/bundle/git-permerror/script
@@ -0,0 +1,25 @@
+mkdir myrepo
+cd myrepo
+cp ../databricks.yml .
+git-repo-init
+mkdir -p subdir/a/b
+
+printf "=== No permission to access .git. Badness: inferred flag is set to true even though we did not infer branch. bundle_root_path is not correct in subdir case.\n"
+trace chmod 000 .git
+errcode trace $CLI bundle validate
+errcode trace $CLI bundle validate -o json | jq .bundle.git
+errcode trace withdir subdir/a/b $CLI bundle validate -o json | jq .bundle.git
+
+printf "\n\n=== No permissions to read .git/HEAD. Badness: warning is not shown. inferred is incorrectly set to true. bundle_root_path is not correct in subdir case.\n"
+chmod 700 .git
+trace chmod 000 .git/HEAD
+errcode trace $CLI bundle validate -o json | jq .bundle.git
+errcode trace withdir subdir/a/b $CLI bundle validate -o json | jq .bundle.git
+
+printf "\n\n=== No permissions to read .git/config. Badness: inferred is incorretly set to true. bundle_root_path is not correct is subdir case.\n"
+chmod 666 .git/HEAD
+trace chmod 000 .git/config
+errcode trace $CLI bundle validate -o json | jq .bundle.git
+errcode trace withdir subdir/a/b $CLI bundle validate -o json | jq .bundle.git
+
+rm -fr .git
diff --git a/acceptance/bundle/git-permerror/test.toml b/acceptance/bundle/git-permerror/test.toml
new file mode 100644
index 000000000..3f96e551c
--- /dev/null
+++ b/acceptance/bundle/git-permerror/test.toml
@@ -0,0 +1,5 @@
+Badness = "Warning logs not shown; inferred flag is set to true incorrect; bundle_root_path is not correct"
+
+[GOOS]
+# This test relies on chmod which does not work on Windows
+windows = false
diff --git a/acceptance/script.prepare b/acceptance/script.prepare
index 87910654d..b814a1260 100644
--- a/acceptance/script.prepare
+++ b/acceptance/script.prepare
@@ -47,3 +47,14 @@ title() {
     local label="$1"
     printf "\n=== %s" "$label"
 }
+
+withdir() {
+    local dir="$1"
+    shift
+    local orig_dir="$(pwd)"
+    cd "$dir" || return $?
+    "$@"
+    local exit_code=$?
+    cd "$orig_dir" || return $?
+    return $exit_code
+}

From b3d98fe66664cb85c750364afce9b1ea0785417f Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 27 Jan 2025 08:45:09 +0100
Subject: [PATCH 146/247] acc: Print replacements on error and rm duplicates
 (#2230)

## Changes
- File comparison files in acceptance test, print the contents of all
applied replacements. Do it once per test.
- Remove duplicate entries in replacement list.

## Tests
Manually, change out files of existing test, you'll get this printed
once, after first assertion:

```
        acceptance_test.go:307: Available replacements:
            REPL /Users/denis\.bilenko/work/cli/acceptance/build/databricks => $$CLI
            REPL /private/var/folders/5y/9kkdnjw91p11vsqwk0cvmk200000gp/T/TestAccept598522733/001 => $$TMPHOME
            ...
```
---
 acceptance/acceptance_test.go | 17 ++++++++++++++---
 libs/testdiff/replacement.go  |  6 +++++-
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index a1c41c5e6..11fd3f2ee 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -228,9 +228,11 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 	formatOutput(out, err)
 	require.NoError(t, out.Close())
 
+	printedRepls := false
+
 	// Compare expected outputs
 	for relPath := range outputs {
-		doComparison(t, repls, dir, tmpDir, relPath)
+		doComparison(t, repls, dir, tmpDir, relPath, &printedRepls)
 	}
 
 	// Make sure there are not unaccounted for new files
@@ -245,12 +247,12 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		if strings.HasPrefix(relPath, "out") {
 			// We have a new file starting with "out"
 			// Show the contents & support overwrite mode for it:
-			doComparison(t, repls, dir, tmpDir, relPath)
+			doComparison(t, repls, dir, tmpDir, relPath, &printedRepls)
 		}
 	}
 }
 
-func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string) {
+func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool) {
 	pathRef := filepath.Join(dirRef, relPath)
 	pathNew := filepath.Join(dirNew, relPath)
 	bufRef, okRef := readIfExists(t, pathRef)
@@ -295,6 +297,15 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN
 		t.Logf("Overwriting existing output file: %s", relPath)
 		testutil.WriteFile(t, pathRef, valueNew)
 	}
+
+	if !equal && printedRepls != nil && !*printedRepls {
+		*printedRepls = true
+		var items []string
+		for _, item := range repls.Repls {
+			items = append(items, fmt.Sprintf("REPL %s => %s", item.Old, item.New))
+		}
+		t.Log("Available replacements:\n" + strings.Join(items, "\n"))
+	}
 }
 
 // Returns combined script.prepare (root) + script.prepare (parent) + ... + script + ... + script.cleanup (parent) + ...
diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go
index 865192662..b512374a3 100644
--- a/libs/testdiff/replacement.go
+++ b/libs/testdiff/replacement.go
@@ -76,7 +76,11 @@ func (r *ReplacementsContext) Set(old, new string) {
 	if err == nil {
 		encodedOld, err := json.Marshal(old)
 		if err == nil {
-			r.appendLiteral(trimQuotes(string(encodedOld)), trimQuotes(string(encodedNew)))
+			encodedStrNew := trimQuotes(string(encodedNew))
+			encodedStrOld := trimQuotes(string(encodedOld))
+			if encodedStrNew != new || encodedStrOld != old {
+				r.appendLiteral(encodedStrOld, encodedStrNew)
+			}
 		}
 	}
 

From 82b0dd36d682b1b11260e05e8a5c6aeccb65c255 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 27 Jan 2025 09:17:22 +0100
Subject: [PATCH 147/247] Add acceptance/selftest, showcasing basic features
 (#2229)

Also make TestInprocessMode use this test.
---
 acceptance/README.md              |  2 ++
 acceptance/acceptance_test.go     |  7 +------
 acceptance/selftest/out.hello.txt |  1 +
 acceptance/selftest/output.txt    | 30 ++++++++++++++++++++++++++++++
 acceptance/selftest/script        | 21 +++++++++++++++++++++
 acceptance/selftest/test.toml     | 11 +++++++++++
 6 files changed, 66 insertions(+), 6 deletions(-)
 create mode 100644 acceptance/selftest/out.hello.txt
 create mode 100644 acceptance/selftest/output.txt
 create mode 100644 acceptance/selftest/script
 create mode 100644 acceptance/selftest/test.toml

diff --git a/acceptance/README.md b/acceptance/README.md
index 42a37d253..75ac1d5fc 100644
--- a/acceptance/README.md
+++ b/acceptance/README.md
@@ -17,3 +17,5 @@ For more complex tests one can also use:
 - `errcode` helper: if the command fails with non-zero code, it appends `Exit code: N` to the output but returns success to caller (bash), allowing continuation of script.
 - `trace` helper: prints the arguments before executing the command.
 - custom output files: redirect output to custom file (it must start with `out`), e.g. `$CLI bundle validate > out.txt 2> out.error.txt`.
+
+See [selftest](./selftest) for a toy test.
diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 11fd3f2ee..6b70c6a7f 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -60,12 +60,7 @@ func TestInprocessMode(t *testing.T) {
 	if InprocessMode {
 		t.Skip("Already tested by TestAccept")
 	}
-	if runtime.GOOS == "windows" {
-		// -  catalogs                               A catalog is the first layer of Unity Catalog’s three-level namespace.
-		// +  catalogs                               A catalog is the first layer of Unity Catalog�s three-level namespace.
-		t.Skip("Fails on CI on unicode characters")
-	}
-	require.NotZero(t, testAccept(t, true, "help"))
+	require.Equal(t, 1, testAccept(t, true, "selftest"))
 }
 
 func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
diff --git a/acceptance/selftest/out.hello.txt b/acceptance/selftest/out.hello.txt
new file mode 100644
index 000000000..e427984d4
--- /dev/null
+++ b/acceptance/selftest/out.hello.txt
@@ -0,0 +1 @@
+HELLO
diff --git a/acceptance/selftest/output.txt b/acceptance/selftest/output.txt
new file mode 100644
index 000000000..d1830e01f
--- /dev/null
+++ b/acceptance/selftest/output.txt
@@ -0,0 +1,30 @@
+=== Capturing STDERR
+>>> python3 -c import sys; sys.stderr.write("STDERR\n")
+STDERR
+
+=== Capturing STDOUT
+>>> python3 -c import sys; sys.stderr.write("STDOUT\n")
+STDOUT
+
+=== Capturing exit code
+>>> errcode python3 -c raise SystemExit(5)
+
+Exit code: 5
+
+=== Capturing exit code (alt)
+>>> python3 -c raise SystemExit(7)
+
+Exit code: 7
+
+=== Capturing pwd
+>>> python3 -c import os; print(os.getcwd())
+$TMPDIR
+
+=== Capturing subdir
+>>> mkdir -p subdir/a/b/c
+
+>>> withdir subdir/a/b/c python3 -c import os; print(os.getcwd())
+$TMPDIR/subdir/a/b/c
+
+=== Custom output files - everything starting with out is captured and compared
+>>> echo HELLO
diff --git a/acceptance/selftest/script b/acceptance/selftest/script
new file mode 100644
index 000000000..89201d925
--- /dev/null
+++ b/acceptance/selftest/script
@@ -0,0 +1,21 @@
+printf "=== Capturing STDERR"
+trace python3 -c 'import sys; sys.stderr.write("STDERR\n")'
+
+printf "\n=== Capturing STDOUT"
+trace python3 -c 'import sys; sys.stderr.write("STDOUT\n")'
+
+printf "\n=== Capturing exit code"
+trace errcode python3 -c 'raise SystemExit(5)'
+
+printf "\n=== Capturing exit code (alt)"
+errcode trace python3 -c 'raise SystemExit(7)'
+
+printf "\n=== Capturing pwd"
+trace python3 -c 'import os; print(os.getcwd())'
+
+printf "\n=== Capturing subdir"
+trace mkdir -p subdir/a/b/c
+trace withdir subdir/a/b/c python3 -c 'import os; print(os.getcwd())' | sed 's/\\/\//g'
+
+printf "\n=== Custom output files - everything starting with out is captured and compared"
+trace echo HELLO > out.hello.txt
diff --git a/acceptance/selftest/test.toml b/acceptance/selftest/test.toml
new file mode 100644
index 000000000..d867a4fd7
--- /dev/null
+++ b/acceptance/selftest/test.toml
@@ -0,0 +1,11 @@
+# Badness = "Brief description of what's wrong with the test output, if anything"
+
+#[GOOS]
+# Disable on Windows
+#windows = false
+
+# Disable on Mac
+#mac = false
+
+# Disable on Linux
+#linux = false

From 1cb32eca907872556b94890e3666ffac531a0f29 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 27 Jan 2025 10:11:06 +0100
Subject: [PATCH 148/247] acc: Support custom replacements (#2231)

## Changes
- Ability to extend a list of replacements via test.toml
- Modify selftest to both demo this feature and to get rid of sed on
Windows.

## Tests
Acceptance tests. I'm also using it
https://github.com/databricks/cli/pull/2213 for things like pid.
---
 acceptance/acceptance_test.go  | 1 +
 acceptance/config_test.go      | 5 +++++
 acceptance/selftest/output.txt | 5 +++++
 acceptance/selftest/script     | 7 ++++++-
 acceptance/selftest/test.toml  | 9 +++++++++
 5 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 6b70c6a7f..e48bd9908 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -190,6 +190,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 	}
 
 	repls.SetPathWithParents(tmpDir, "$TMPDIR")
+	repls.Repls = append(repls.Repls, config.Repls...)
 
 	scriptContents := readMergedScriptContents(t, dir)
 	testutil.WriteFile(t, filepath.Join(tmpDir, EntryPointScript), scriptContents)
diff --git a/acceptance/config_test.go b/acceptance/config_test.go
index 49dce06ba..41866c4a7 100644
--- a/acceptance/config_test.go
+++ b/acceptance/config_test.go
@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/BurntSushi/toml"
+	"github.com/databricks/cli/libs/testdiff"
 	"github.com/stretchr/testify/require"
 )
 
@@ -24,6 +25,10 @@ type TestConfig struct {
 	// Which OSes the test is enabled on. Each string is compared against runtime.GOOS.
 	// If absent, default to true.
 	GOOS map[string]bool
+
+	// List of additional replacements to apply on this test.
+	// Old is a regexp, New is a replacement expression.
+	Repls []testdiff.Replacement
 }
 
 // FindConfig finds the closest config file.
diff --git a/acceptance/selftest/output.txt b/acceptance/selftest/output.txt
index d1830e01f..9fdfbc1e7 100644
--- a/acceptance/selftest/output.txt
+++ b/acceptance/selftest/output.txt
@@ -28,3 +28,8 @@ $TMPDIR/subdir/a/b/c
 
 === Custom output files - everything starting with out is captured and compared
 >>> echo HELLO
+
+=== Custom regex can be specified in [[Repl]] section
+1234
+CUSTOM_NUMBER_REGEX
+123456
diff --git a/acceptance/selftest/script b/acceptance/selftest/script
index 89201d925..665726167 100644
--- a/acceptance/selftest/script
+++ b/acceptance/selftest/script
@@ -15,7 +15,12 @@ trace python3 -c 'import os; print(os.getcwd())'
 
 printf "\n=== Capturing subdir"
 trace mkdir -p subdir/a/b/c
-trace withdir subdir/a/b/c python3 -c 'import os; print(os.getcwd())' | sed 's/\\/\//g'
+trace withdir subdir/a/b/c python3 -c 'import os; print(os.getcwd())'
 
 printf "\n=== Custom output files - everything starting with out is captured and compared"
 trace echo HELLO > out.hello.txt
+
+printf "\n=== Custom regex can be specified in [[Repl]] section\n"
+echo 1234
+echo 12345
+echo 123456
diff --git a/acceptance/selftest/test.toml b/acceptance/selftest/test.toml
index d867a4fd7..9607ec5df 100644
--- a/acceptance/selftest/test.toml
+++ b/acceptance/selftest/test.toml
@@ -9,3 +9,12 @@
 
 # Disable on Linux
 #linux = false
+
+[[Repls]]
+Old = '\b[0-9]{5}\b'
+New = "CUSTOM_NUMBER_REGEX"
+
+[[Repls]]
+# Fix path with reverse slashes in the output for Windows.
+Old = '\$TMPDIR\\subdir\\a\\b\\c'
+New = '$$TMPDIR/subdir/a/b/c'

From 6e8f0ea8afeecf86c3edd42d0ccccbacf25353d2 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 27 Jan 2025 11:33:16 +0100
Subject: [PATCH 149/247] CI: Move ruff to 'lint' job (#2232)

This is where it belongs and also there is no need to run it 3 times.
---
 .github/workflows/push.yml | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
index c3a314d69..2a8a68862 100644
--- a/.github/workflows/push.yml
+++ b/.github/workflows/push.yml
@@ -60,12 +60,6 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@887a942a15af3a7626099df99e897a18d9e5ab3a # v5.1.0
 
-      - name: Run ruff
-        uses: astral-sh/ruff-action@31a518504640beb4897d0b9f9e50a2a9196e75ba # v3.0.1
-        with:
-          version: "0.9.1"
-          args: "format --check"
-
       - name: Set go env
         run: |
           echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
@@ -80,7 +74,7 @@ jobs:
       - name: Run tests with coverage
         run: make cover
 
-  golangci:
+  linters:
     needs: cleanups
     name: lint
     runs-on: ubuntu-latest
@@ -105,6 +99,11 @@ jobs:
         with:
           version: v1.63.4
           args: --timeout=15m
+      - name: Run ruff
+        uses: astral-sh/ruff-action@31a518504640beb4897d0b9f9e50a2a9196e75ba # v3.0.1
+        with:
+          version: "0.9.1"
+          args: "format --check"
 
   validate-bundle-schema:
     needs: cleanups

From b7dd70b8b3c59d64ab7b54805750b532b0d75f07 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 27 Jan 2025 13:22:40 +0100
Subject: [PATCH 150/247] acc: Add a couple of error tests for 'bundle init'
 (#2233)

This captures how we log errors related to subprocess run and what does
the output look like.
---
 acceptance/bundle/templates/wrong-path/output.txt | 3 +++
 acceptance/bundle/templates/wrong-path/script     | 2 ++
 acceptance/bundle/templates/wrong-path/test.toml  | 1 +
 acceptance/bundle/templates/wrong-url/output.txt  | 5 +++++
 acceptance/bundle/templates/wrong-url/script      | 2 ++
 acceptance/bundle/templates/wrong-url/test.toml   | 7 +++++++
 6 files changed, 20 insertions(+)
 create mode 100644 acceptance/bundle/templates/wrong-path/output.txt
 create mode 100644 acceptance/bundle/templates/wrong-path/script
 create mode 100644 acceptance/bundle/templates/wrong-path/test.toml
 create mode 100644 acceptance/bundle/templates/wrong-url/output.txt
 create mode 100644 acceptance/bundle/templates/wrong-url/script
 create mode 100644 acceptance/bundle/templates/wrong-url/test.toml

diff --git a/acceptance/bundle/templates/wrong-path/output.txt b/acceptance/bundle/templates/wrong-path/output.txt
new file mode 100644
index 000000000..0a6fdfc84
--- /dev/null
+++ b/acceptance/bundle/templates/wrong-path/output.txt
@@ -0,0 +1,3 @@
+Error: not a bundle template: expected to find a template schema file at databricks_template_schema.json
+
+Exit code: 1
diff --git a/acceptance/bundle/templates/wrong-path/script b/acceptance/bundle/templates/wrong-path/script
new file mode 100644
index 000000000..00c05927a
--- /dev/null
+++ b/acceptance/bundle/templates/wrong-path/script
@@ -0,0 +1,2 @@
+export NO_COLOR=1
+$CLI bundle init /DOES/NOT/EXIST
diff --git a/acceptance/bundle/templates/wrong-path/test.toml b/acceptance/bundle/templates/wrong-path/test.toml
new file mode 100644
index 000000000..4bbcb5100
--- /dev/null
+++ b/acceptance/bundle/templates/wrong-path/test.toml
@@ -0,0 +1 @@
+Badness = 'The error message should include full path: "expected to find a template schema file at databricks_template_schema.json"'
diff --git a/acceptance/bundle/templates/wrong-url/output.txt b/acceptance/bundle/templates/wrong-url/output.txt
new file mode 100644
index 000000000..b78cf4b68
--- /dev/null
+++ b/acceptance/bundle/templates/wrong-url/output.txt
@@ -0,0 +1,5 @@
+Error: git clone failed: git clone https://invalid-domain-123.databricks.com/hello/world $TMPDIR_GPARENT/world-123456 --no-tags --depth=1: exit status 128. Cloning into '$TMPDIR_GPARENT/world-123456'...
+fatal: unable to access 'https://invalid-domain-123.databricks.com/hello/world/': Could not resolve host: invalid-domain-123.databricks.com
+
+
+Exit code: 1
diff --git a/acceptance/bundle/templates/wrong-url/script b/acceptance/bundle/templates/wrong-url/script
new file mode 100644
index 000000000..e9bc0f4f6
--- /dev/null
+++ b/acceptance/bundle/templates/wrong-url/script
@@ -0,0 +1,2 @@
+export NO_COLOR=1
+$CLI bundle init https://invalid-domain-123.databricks.com/hello/world
diff --git a/acceptance/bundle/templates/wrong-url/test.toml b/acceptance/bundle/templates/wrong-url/test.toml
new file mode 100644
index 000000000..0bb24bf1a
--- /dev/null
+++ b/acceptance/bundle/templates/wrong-url/test.toml
@@ -0,0 +1,7 @@
+[[Repls]]
+Old = '\\'
+New = '/'
+
+[[Repls]]
+Old = '/world-[0-9]+'
+New = '/world-123456'

From 4595c6f1b5d4890b6c9a1e13257319d52954dfe5 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 27 Jan 2025 14:11:07 +0100
Subject: [PATCH 151/247] Bump github.com/databricks/databricks-sdk-go from
 0.55.0 to 0.56.1 (#2238)

Bumps
[github.com/databricks/databricks-sdk-go](https://github.com/databricks/databricks-sdk-go)
from 0.55.0 to 0.56.1.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/databricks/databricks-sdk-go/releases">github.com/databricks/databricks-sdk-go's
releases</a>.</em></p>
<blockquote>
<h2>v0.56.1</h2>
<h3>Bug Fixes</h3>
<ul>
<li>Do not send query parameters when set to zero value (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1136">#1136</a>).</li>
</ul>
<h2>v0.56.0</h2>
<h3>Bug Fixes</h3>
<ul>
<li>Support Query parameters for all HTTP operations (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1124">#1124</a>).</li>
</ul>
<h3>Internal Changes</h3>
<ul>
<li>Add download target to MakeFile (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1125">#1125</a>).</li>
<li>Delete examples/mocking module (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1126">#1126</a>).</li>
<li>Scope the traversing directory in the Recursive list workspace test
(<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1120">#1120</a>).</li>
</ul>
<h3>API Changes:</h3>
<ul>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/iam#AccessControlAPI">w.AccessControl</a>
workspace-level service.</li>
<li>Added <code>HttpRequest</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service.</li>
<li>Added <code>ReviewState</code>, <code>Reviews</code> and
<code>RunnerCollaborators</code> fields for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/cleanrooms#CleanRoomAssetNotebook">cleanrooms.CleanRoomAssetNotebook</a>.</li>
<li>Added <code>CleanRoomsNotebookOutput</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#RunOutput">jobs.RunOutput</a>.</li>
<li>Added <code>RunAsRepl</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#SparkJarTask">jobs.SparkJarTask</a>.</li>
<li>Added <code>Scopes</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/oauth2#UpdateCustomAppIntegration">oauth2.UpdateCustomAppIntegration</a>.</li>
<li>Added <code>Contents</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#GetOpenApiResponse">serving.GetOpenApiResponse</a>.</li>
<li>Added <code>Activated</code>, <code>ActivationUrl</code>,
<code>AuthenticationType</code>, <code>Cloud</code>,
<code>Comment</code>, <code>CreatedAt</code>, <code>CreatedBy</code>,
<code>DataRecipientGlobalMetastoreId</code>, <code>IpAccessList</code>,
<code>MetastoreId</code>, <code>Name</code>, <code>Owner</code>,
<code>PropertiesKvpairs</code>, <code>Region</code>,
<code>SharingCode</code>, <code>Tokens</code>, <code>UpdatedAt</code>
and <code>UpdatedBy</code> fields for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/sharing#RecipientInfo">sharing.RecipientInfo</a>.</li>
<li>Added <code>ExpirationTime</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/sharing#RecipientInfo">sharing.RecipientInfo</a>.</li>
<li>Added <code>Pending</code> enum value for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/cleanrooms#CleanRoomAssetStatusEnum">cleanrooms.CleanRoomAssetStatusEnum</a>.</li>
<li>Added <code>AddNodesFailed</code>,
<code>AutomaticClusterUpdate</code>, <code>AutoscalingBackoff</code> and
<code>AutoscalingFailed</code> enum values for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#EventType">compute.EventType</a>.</li>
<li>Added <code>PendingWarehouse</code> enum value for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/dashboards#MessageStatus">dashboards.MessageStatus</a>.</li>
<li>Added <code>Cpu</code>, <code>GpuLarge</code>,
<code>GpuMedium</code>, <code>GpuSmall</code> and
<code>MultigpuMedium</code> enum values for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingModelWorkloadType">serving.ServingModelWorkloadType</a>.</li>
<li>Changed <code>Update</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/sharing#RecipientsAPI">w.Recipients</a>
workspace-level service to return <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/sharing#RecipientInfo">sharing.RecipientInfo</a>.</li>
<li>Changed <code>Update</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/sharing#RecipientsAPI">w.Recipients</a>
workspace-level service return type to become non-empty.</li>
<li>Changed <code>Update</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/sharing#RecipientsAPI">w.Recipients</a>
workspace-level service to type <code>Update</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/sharing#RecipientsAPI">w.Recipients</a>
workspace-level service.</li>
<li>Changed <code>Create</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service with new required argument order.</li>
<li>Changed <code>GetOpenApi</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service return type to become non-empty.</li>
<li>Changed <code>Patch</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service to type <code>Patch</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service.</li>
<li>Changed <code>Patch</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service to return <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#EndpointTags">serving.EndpointTags</a>.</li>
<li>Changed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#EndpointTagList">serving.EndpointTagList</a>
to.</li>
<li>Changed <code>CollaboratorAlias</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/cleanrooms#CleanRoomCollaborator">cleanrooms.CleanRoomCollaborator</a>
to be required.</li>
<li>Changed <code>CollaboratorAlias</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/cleanrooms#CleanRoomCollaborator">cleanrooms.CleanRoomCollaborator</a>
to be required.</li>
<li>Changed <code>Behavior</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AiGatewayGuardrailPiiBehavior">serving.AiGatewayGuardrailPiiBehavior</a>
to no longer be required.</li>
<li>Changed <code>Behavior</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AiGatewayGuardrailPiiBehavior">serving.AiGatewayGuardrailPiiBehavior</a>
to no longer be required.</li>
<li>Changed <code>Config</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#CreateServingEndpoint">serving.CreateServingEndpoint</a>
to no longer be required.</li>
<li>Changed <code>ProjectId</code> and <code>Region</code> fields for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#GoogleCloudVertexAiConfig">serving.GoogleCloudVertexAiConfig</a>
to be required.</li>
<li>Changed <code>ProjectId</code> and <code>Region</code> fields for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#GoogleCloudVertexAiConfig">serving.GoogleCloudVertexAiConfig</a>
to be required.</li>
<li>Changed <code>WorkloadType</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServedEntityInput">serving.ServedEntityInput</a>
to type <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingModelWorkloadType">serving.ServingModelWorkloadType</a>.</li>
<li>Changed <code>WorkloadType</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServedEntityOutput">serving.ServedEntityOutput</a>
to type <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingModelWorkloadType">serving.ServingModelWorkloadType</a>.</li>
</ul>
<!-- raw HTML omitted -->
</blockquote>
<p>... (truncated)</p>
</details>
<details>
<summary>Changelog</summary>
<p><em>Sourced from <a
href="https://github.com/databricks/databricks-sdk-go/blob/main/CHANGELOG.md">github.com/databricks/databricks-sdk-go's
changelog</a>.</em></p>
<blockquote>
<h2>[Release] Release v0.56.1</h2>
<h3>Bug Fixes</h3>
<ul>
<li>Do not send query parameters when set to zero value (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1136">#1136</a>).</li>
</ul>
<h2>[Release] Release v0.56.0</h2>
<h3>Bug Fixes</h3>
<ul>
<li>Support Query parameters for all HTTP operations (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1124">#1124</a>).</li>
</ul>
<h3>Internal Changes</h3>
<ul>
<li>Add download target to MakeFile (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1125">#1125</a>).</li>
<li>Delete examples/mocking module (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1126">#1126</a>).</li>
<li>Scope the traversing directory in the Recursive list workspace test
(<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1120">#1120</a>).</li>
</ul>
<h3>API Changes:</h3>
<ul>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/iam#AccessControlAPI">w.AccessControl</a>
workspace-level service.</li>
<li>Added <code>HttpRequest</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service.</li>
<li>Added <code>ReviewState</code>, <code>Reviews</code> and
<code>RunnerCollaborators</code> fields for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/cleanrooms#CleanRoomAssetNotebook">cleanrooms.CleanRoomAssetNotebook</a>.</li>
<li>Added <code>CleanRoomsNotebookOutput</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#RunOutput">jobs.RunOutput</a>.</li>
<li>Added <code>RunAsRepl</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#SparkJarTask">jobs.SparkJarTask</a>.</li>
<li>Added <code>Scopes</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/oauth2#UpdateCustomAppIntegration">oauth2.UpdateCustomAppIntegration</a>.</li>
<li>Added <code>Contents</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#GetOpenApiResponse">serving.GetOpenApiResponse</a>.</li>
<li>Added <code>Activated</code>, <code>ActivationUrl</code>,
<code>AuthenticationType</code>, <code>Cloud</code>,
<code>Comment</code>, <code>CreatedAt</code>, <code>CreatedBy</code>,
<code>DataRecipientGlobalMetastoreId</code>, <code>IpAccessList</code>,
<code>MetastoreId</code>, <code>Name</code>, <code>Owner</code>,
<code>PropertiesKvpairs</code>, <code>Region</code>,
<code>SharingCode</code>, <code>Tokens</code>, <code>UpdatedAt</code>
and <code>UpdatedBy</code> fields for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/sharing#RecipientInfo">sharing.RecipientInfo</a>.</li>
<li>Added <code>ExpirationTime</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/sharing#RecipientInfo">sharing.RecipientInfo</a>.</li>
<li>Added <code>Pending</code> enum value for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/cleanrooms#CleanRoomAssetStatusEnum">cleanrooms.CleanRoomAssetStatusEnum</a>.</li>
<li>Added <code>AddNodesFailed</code>,
<code>AutomaticClusterUpdate</code>, <code>AutoscalingBackoff</code> and
<code>AutoscalingFailed</code> enum values for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#EventType">compute.EventType</a>.</li>
<li>Added <code>PendingWarehouse</code> enum value for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/dashboards#MessageStatus">dashboards.MessageStatus</a>.</li>
<li>Added <code>Cpu</code>, <code>GpuLarge</code>,
<code>GpuMedium</code>, <code>GpuSmall</code> and
<code>MultigpuMedium</code> enum values for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingModelWorkloadType">serving.ServingModelWorkloadType</a>.</li>
<li>Changed <code>Update</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/sharing#RecipientsAPI">w.Recipients</a>
workspace-level service to return <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/sharing#RecipientInfo">sharing.RecipientInfo</a>.</li>
<li>Changed <code>Update</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/sharing#RecipientsAPI">w.Recipients</a>
workspace-level service return type to become non-empty.</li>
<li>Changed <code>Update</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/sharing#RecipientsAPI">w.Recipients</a>
workspace-level service to type <code>Update</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/sharing#RecipientsAPI">w.Recipients</a>
workspace-level service.</li>
<li>Changed <code>Create</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service with new required argument order.</li>
<li>Changed <code>GetOpenApi</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service return type to become non-empty.</li>
<li>Changed <code>Patch</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service to type <code>Patch</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service.</li>
<li>Changed <code>Patch</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service to return <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#EndpointTags">serving.EndpointTags</a>.</li>
<li>Changed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#EndpointTagList">serving.EndpointTagList</a>
to.</li>
<li>Changed <code>CollaboratorAlias</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/cleanrooms#CleanRoomCollaborator">cleanrooms.CleanRoomCollaborator</a>
to be required.</li>
<li>Changed <code>CollaboratorAlias</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/cleanrooms#CleanRoomCollaborator">cleanrooms.CleanRoomCollaborator</a>
to be required.</li>
<li>Changed <code>Behavior</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AiGatewayGuardrailPiiBehavior">serving.AiGatewayGuardrailPiiBehavior</a>
to no longer be required.</li>
<li>Changed <code>Behavior</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AiGatewayGuardrailPiiBehavior">serving.AiGatewayGuardrailPiiBehavior</a>
to no longer be required.</li>
<li>Changed <code>Config</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#CreateServingEndpoint">serving.CreateServingEndpoint</a>
to no longer be required.</li>
</ul>
<!-- raw HTML omitted -->
</blockquote>
<p>... (truncated)</p>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/bf617bb7a6f46370b94886dd674e4721b17224fd"><code>bf617bb</code></a>
[Release] Release v0.56.1 (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/1137">#1137</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/18cebf1d5ca8889ae82f660c96fecc8bc5b73be5"><code>18cebf1</code></a>
[Fix] Do not send query parameters when set to zero value (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/1136">#1136</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/28ff749ee2271172ceda01aaaa6e997e8c2aebd7"><code>28ff749</code></a>
[Release] Release v0.56.0 (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/1134">#1134</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/113454080f34e4da04782895ea5d61101bf2b425"><code>1134540</code></a>
[Internal] Add download target to MakeFile (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/1125">#1125</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/e079db96f33d53d6659b222a905da366dbab576b"><code>e079db9</code></a>
[Fix] Support Query parameters for all HTTP operations (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/1124">#1124</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/1045fb9697db505f5fd1ca0ebe4be8b6479df981"><code>1045fb9</code></a>
[Internal] Delete examples/mocking module (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/1126">#1126</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/914ab6b7e8e48ca6da6803c10c2d720ba496cd87"><code>914ab6b</code></a>
[Internal] Scope the traversing directory in the Recursive list
workspace tes...</li>
<li>See full diff in <a
href="https://github.com/databricks/databricks-sdk-go/compare/v0.55.0...v0.56.1">compare
view</a></li>
</ul>
</details>
<br />

<details>
<summary>Most Recent Ignore Conditions Applied to This Pull
Request</summary>

| Dependency Name | Ignore Conditions |
| --- | --- |
| github.com/databricks/databricks-sdk-go | [>= 0.28.a, < 0.29] |
</details>


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/databricks/databricks-sdk-go&package-manager=go_modules&previous-version=0.55.0&new-version=0.56.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Andrew Nester <andrew.nester@databricks.com>
---
 .codegen/_openapi_sha                         |   2 +-
 .codegen/service.go.tmpl                      |  20 +-
 .gitattributes                                |   1 +
 bundle/deploy/terraform/convert_test.go       |   4 +-
 .../convert_model_serving_endpoint_test.go    |   2 +-
 .../internal/schema/annotations_openapi.yml   | 367 +++++++++++-------
 .../schema/annotations_openapi_overrides.yml  |  11 +
 bundle/schema/jsonschema.json                 | 175 +++++----
 .../custom-app-integration.go                 |   1 +
 cmd/api/api.go                                |   2 +-
 .../access-control/access-control.go          | 109 ++++++
 cmd/workspace/cmd.go                          |   2 +
 cmd/workspace/providers/providers.go          |   4 +-
 cmd/workspace/recipients/recipients.go        |  96 ++---
 .../serving-endpoints/serving-endpoints.go    | 111 +++++-
 go.mod                                        |   2 +-
 go.sum                                        |   4 +-
 integration/cmd/sync/sync_test.go             |   2 +-
 libs/filer/files_client.go                    |   4 +-
 libs/filer/workspace_files_client.go          |   5 +-
 .../workspace_files_extensions_client_test.go |   2 +-
 libs/git/info.go                              |   1 +
 22 files changed, 588 insertions(+), 339 deletions(-)
 create mode 100755 cmd/workspace/access-control/access-control.go

diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha
index dfe78790a..588cf9d63 100644
--- a/.codegen/_openapi_sha
+++ b/.codegen/_openapi_sha
@@ -1 +1 @@
-779817ed8d63031f5ea761fbd25ee84f38feec0d
\ No newline at end of file
+0be1b914249781b5e903b7676fd02255755bc851
\ No newline at end of file
diff --git a/.codegen/service.go.tmpl b/.codegen/service.go.tmpl
index 0c9fa089a..2f4987b13 100644
--- a/.codegen/service.go.tmpl
+++ b/.codegen/service.go.tmpl
@@ -109,16 +109,19 @@ var {{.CamelName}}Overrides []func(
 	{{- end }}
 )
 
+{{- $excludeFromJson := list "http-request"}}
+
 func new{{.PascalName}}() *cobra.Command {
 	cmd := &cobra.Command{}
 
+	{{- $canUseJson := and .CanUseJson (not (in $excludeFromJson .KebabName )) -}}
 	{{- if .Request}}
 
 	var {{.CamelName}}Req {{.Service.Package.Name}}.{{.Request.PascalName}}
 	{{- if .RequestBodyField }}
 	{{.CamelName}}Req.{{.RequestBodyField.PascalName}} = &{{.Service.Package.Name}}.{{.RequestBodyField.Entity.PascalName}}{}
 	{{- end }}
-	{{- if .CanUseJson}}
+	{{- if $canUseJson}}
 	var {{.CamelName}}Json flags.JsonFlag
 	{{- end}}
 	{{- end}}
@@ -135,7 +138,7 @@ func new{{.PascalName}}() *cobra.Command {
 	{{- $request = .RequestBodyField.Entity -}}
 	{{- end -}}
     {{if $request }}// TODO: short flags
-	{{- if .CanUseJson}}
+	{{- if $canUseJson}}
 	cmd.Flags().Var(&{{.CamelName}}Json, "json", `either inline JSON string or @path/to/file.json with request body`)
 	{{- end}}
     {{$method := .}}
@@ -177,7 +180,7 @@ func new{{.PascalName}}() *cobra.Command {
 	{{- $hasRequiredArgs :=  and (not $hasIdPrompt) $hasPosArgs -}}
 	{{- $hasSingleRequiredRequestBodyFieldWithPrompt := and (and $hasIdPrompt $request) (eq 1 (len $request.RequiredRequestBodyFields))  -}}
 	{{- $onlyPathArgsRequiredAsPositionalArguments := and $request (eq (len .RequiredPositionalArguments) (len $request.RequiredPathFields)) -}}
-	{{- $hasDifferentArgsWithJsonFlag := and (not $onlyPathArgsRequiredAsPositionalArguments) (and .CanUseJson (or $request.HasRequiredRequestBodyFields )) -}}
+	{{- $hasDifferentArgsWithJsonFlag := and (not $onlyPathArgsRequiredAsPositionalArguments) (and $canUseJson (or $request.HasRequiredRequestBodyFields )) -}}
 	{{- $hasCustomArgHandler := or $hasRequiredArgs $hasDifferentArgsWithJsonFlag -}}
 
 	{{- $atleastOneArgumentWithDescription := false -}}
@@ -239,7 +242,7 @@ func new{{.PascalName}}() *cobra.Command {
 		ctx := cmd.Context()
 		{{if .Service.IsAccounts}}a := root.AccountClient(ctx){{else}}w := root.WorkspaceClient(ctx){{end}}
 		{{- if .Request }}
-			{{ if .CanUseJson }}
+			{{ if $canUseJson }}
 			if cmd.Flags().Changed("json") {
 					diags := {{.CamelName}}Json.Unmarshal(&{{.CamelName}}Req{{ if .RequestBodyField }}.{{.RequestBodyField.PascalName}}{{ end }})
 					if diags.HasError() {
@@ -255,7 +258,7 @@ func new{{.PascalName}}() *cobra.Command {
 				return fmt.Errorf("please provide command input in JSON format by specifying the --json flag")
 			}{{- end}}
 			{{- if $hasPosArgs }}
-			{{- if and .CanUseJson $hasSingleRequiredRequestBodyFieldWithPrompt }} else {
+			{{- if and $canUseJson $hasSingleRequiredRequestBodyFieldWithPrompt }} else {
 			{{- end}}
 			{{- if $hasIdPrompt}}
 				if len(args) == 0 {
@@ -279,9 +282,9 @@ func new{{.PascalName}}() *cobra.Command {
 
 			{{$method := .}}
 			{{- range $arg, $field := .RequiredPositionalArguments}}
-				{{- template "args-scan" (dict "Arg" $arg "Field" $field "Method" $method "HasIdPrompt" $hasIdPrompt)}}
+				{{- template "args-scan" (dict "Arg" $arg "Field" $field "Method" $method "HasIdPrompt" $hasIdPrompt "ExcludeFromJson" $excludeFromJson)}}
 			{{- end -}}
-			{{- if and .CanUseJson $hasSingleRequiredRequestBodyFieldWithPrompt }}
+			{{- if and $canUseJson $hasSingleRequiredRequestBodyFieldWithPrompt }}
 			}
 			{{- end}}
 
@@ -392,7 +395,8 @@ func new{{.PascalName}}() *cobra.Command {
 	{{- $method := .Method -}}
 	{{- $arg := .Arg -}}
 	{{- $hasIdPrompt := .HasIdPrompt -}}
-	{{- $optionalIfJsonIsUsed := and (not $hasIdPrompt) (and $field.IsRequestBodyField $method.CanUseJson) }}
+	{{ $canUseJson := and $method.CanUseJson (not (in .ExcludeFromJson $method.KebabName)) }}
+	{{- $optionalIfJsonIsUsed := and (not $hasIdPrompt) (and $field.IsRequestBodyField $canUseJson) }}
 	{{- if $optionalIfJsonIsUsed  }}
 	if !cmd.Flags().Changed("json") {
 	{{- end }}
diff --git a/.gitattributes b/.gitattributes
index 0a8ddf3cb..ebe94ed8e 100755
--- a/.gitattributes
+++ b/.gitattributes
@@ -31,6 +31,7 @@ cmd/account/users/users.go linguist-generated=true
 cmd/account/vpc-endpoints/vpc-endpoints.go linguist-generated=true
 cmd/account/workspace-assignment/workspace-assignment.go linguist-generated=true
 cmd/account/workspaces/workspaces.go linguist-generated=true
+cmd/workspace/access-control/access-control.go linguist-generated=true
 cmd/workspace/aibi-dashboard-embedding-access-policy/aibi-dashboard-embedding-access-policy.go linguist-generated=true
 cmd/workspace/aibi-dashboard-embedding-approved-domains/aibi-dashboard-embedding-approved-domains.go linguist-generated=true
 cmd/workspace/alerts-legacy/alerts-legacy.go linguist-generated=true
diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go
index ffe55db71..afc1fb22a 100644
--- a/bundle/deploy/terraform/convert_test.go
+++ b/bundle/deploy/terraform/convert_test.go
@@ -419,7 +419,7 @@ func TestBundleToTerraformModelServing(t *testing.T) {
 	src := resources.ModelServingEndpoint{
 		CreateServingEndpoint: &serving.CreateServingEndpoint{
 			Name: "name",
-			Config: serving.EndpointCoreConfigInput{
+			Config: &serving.EndpointCoreConfigInput{
 				ServedModels: []serving.ServedModelInput{
 					{
 						ModelName:          "model_name",
@@ -474,7 +474,7 @@ func TestBundleToTerraformModelServingPermissions(t *testing.T) {
 			// and as such observed the `omitempty` tag.
 			// The new method leverages [dyn.Value] where any field that is not
 			// explicitly set is not part of the value.
-			Config: serving.EndpointCoreConfigInput{
+			Config: &serving.EndpointCoreConfigInput{
 				ServedModels: []serving.ServedModelInput{
 					{
 						ModelName:          "model_name",
diff --git a/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go b/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go
index d46350bb7..98cf2dc22 100644
--- a/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go
+++ b/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go
@@ -17,7 +17,7 @@ func TestConvertModelServingEndpoint(t *testing.T) {
 	src := resources.ModelServingEndpoint{
 		CreateServingEndpoint: &serving.CreateServingEndpoint{
 			Name: "name",
-			Config: serving.EndpointCoreConfigInput{
+			Config: &serving.EndpointCoreConfigInput{
 				ServedModels: []serving.ServedModelInput{
 					{
 						ModelName:          "model_name",
diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml
index 8ff5c9253..d5a9bf69e 100644
--- a/bundle/internal/schema/annotations_openapi.yml
+++ b/bundle/internal/schema/annotations_openapi.yml
@@ -353,12 +353,12 @@ github.com/databricks/cli/bundle/config/resources.MlflowModel:
 github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint:
   "ai_gateway":
     "description": |-
-      The AI Gateway configuration for the serving endpoint. NOTE: only external model endpoints are supported as of now.
+      The AI Gateway configuration for the serving endpoint. NOTE: Only external model and provisioned throughput endpoints are currently supported.
   "config":
     "description": |-
       The core config of the serving endpoint.
   "name":
-    "description": |
+    "description": |-
       The name of the serving endpoint. This field is required and must be unique across a Databricks workspace.
       An endpoint name can consist of alphanumeric characters, dashes, and underscores.
   "rate_limits":
@@ -1974,6 +1974,9 @@ github.com/databricks/databricks-sdk-go/service/jobs.SparkJarTask:
       Parameters passed to the main method.
 
       Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.
+  "run_as_repl":
+    "description": |-
+      Deprecated. A value of `false` is no longer supported.
 github.com/databricks/databricks-sdk-go/service/jobs.SparkPythonTask:
   "parameters":
     "description": |-
@@ -2684,27 +2687,36 @@ github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfigScd
 github.com/databricks/databricks-sdk-go/service/serving.Ai21LabsConfig:
   "ai21labs_api_key":
     "description": |-
-      The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`.
+      The Databricks secret key reference for an AI21 Labs API key. If you
+      prefer to paste your API key directly, see `ai21labs_api_key_plaintext`.
+      You must provide an API key using one of the following fields:
+      `ai21labs_api_key` or `ai21labs_api_key_plaintext`.
   "ai21labs_api_key_plaintext":
     "description": |-
-      An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`.
+      An AI21 Labs API key provided as a plaintext string. If you prefer to
+      reference your key using Databricks Secrets, see `ai21labs_api_key`. You
+      must provide an API key using one of the following fields:
+      `ai21labs_api_key` or `ai21labs_api_key_plaintext`.
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayConfig:
   "guardrails":
     "description": |-
       Configuration for AI Guardrails to prevent unwanted data and unsafe data in requests and responses.
   "inference_table_config":
     "description": |-
-      Configuration for payload logging using inference tables. Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.
+      Configuration for payload logging using inference tables.
+      Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.
   "rate_limits":
     "description": |-
       Configuration for rate limits which can be set to limit endpoint traffic.
   "usage_tracking_config":
     "description": |-
-      Configuration to enable usage tracking using system tables. These tables allow you to monitor operational usage on endpoints and their associated costs.
+      Configuration to enable usage tracking using system tables.
+      These tables allow you to monitor operational usage on endpoints and their associated costs.
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParameters:
   "invalid_keywords":
     "description": |-
-      List of invalid keywords. AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.
+      List of invalid keywords.
+      AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.
   "pii":
     "description": |-
       Configuration for guardrail PII filter.
@@ -2713,15 +2725,14 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParame
       Indicates whether the safety filter is enabled.
   "valid_topics":
     "description": |-
-      The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics.
+      The list of allowed topics.
+      Given a chat request, this guardrail flags the request if its topic is not in the allowed topics.
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehavior:
   "behavior":
     "description": |-
-      Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.
+      Configuration for input guardrail filters.
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehaviorBehavior:
   "_":
-    "description": |-
-      Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.
     "enum":
       - |-
         NONE
@@ -2737,30 +2748,32 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrails:
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayInferenceTableConfig:
   "catalog_name":
     "description": |-
-      The name of the catalog in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the catalog name.
+      The name of the catalog in Unity Catalog. Required when enabling inference tables.
+      NOTE: On update, you have to disable inference table first in order to change the catalog name.
   "enabled":
     "description": |-
       Indicates whether the inference table is enabled.
   "schema_name":
     "description": |-
-      The name of the schema in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the schema name.
+      The name of the schema in Unity Catalog. Required when enabling inference tables.
+      NOTE: On update, you have to disable inference table first in order to change the schema name.
   "table_name_prefix":
     "description": |-
-      The prefix of the table in Unity Catalog. NOTE: On update, you have to disable inference table first in order to change the prefix name.
+      The prefix of the table in Unity Catalog.
+      NOTE: On update, you have to disable inference table first in order to change the prefix name.
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit:
   "calls":
     "description": |-
       Used to specify how many calls are allowed for a key within the renewal_period.
   "key":
     "description": |-
-      Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.
+      Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported,
+      with 'endpoint' being the default if not specified.
   "renewal_period":
     "description": |-
       Renewal period field for a rate limit. Currently, only 'minute' is supported.
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitKey:
   "_":
-    "description": |-
-      Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.
     "enum":
       - |-
         user
@@ -2768,8 +2781,6 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitKey:
         endpoint
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitRenewalPeriod:
   "_":
-    "description": |-
-      Renewal period field for a rate limit. Currently, only 'minute' is supported.
     "enum":
       - |-
         minute
@@ -2780,26 +2791,43 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayUsageTrackingCo
 github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfig:
   "aws_access_key_id":
     "description": |-
-      The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.
+      The Databricks secret key reference for an AWS access key ID with
+      permissions to interact with Bedrock services. If you prefer to paste
+      your API key directly, see `aws_access_key_id_plaintext`. You must provide an API
+      key using one of the following fields: `aws_access_key_id` or
+      `aws_access_key_id_plaintext`.
   "aws_access_key_id_plaintext":
     "description": |-
-      An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.
+      An AWS access key ID with permissions to interact with Bedrock services
+      provided as a plaintext string. If you prefer to reference your key using
+      Databricks Secrets, see `aws_access_key_id`. You must provide an API key
+      using one of the following fields: `aws_access_key_id` or
+      `aws_access_key_id_plaintext`.
   "aws_region":
     "description": |-
       The AWS region to use. Bedrock has to be enabled there.
   "aws_secret_access_key":
     "description": |-
-      The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.
+      The Databricks secret key reference for an AWS secret access key paired
+      with the access key ID, with permissions to interact with Bedrock
+      services. If you prefer to paste your API key directly, see
+      `aws_secret_access_key_plaintext`. You must provide an API key using one
+      of the following fields: `aws_secret_access_key` or
+      `aws_secret_access_key_plaintext`.
   "aws_secret_access_key_plaintext":
     "description": |-
-      An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.
+      An AWS secret access key paired with the access key ID, with permissions
+      to interact with Bedrock services provided as a plaintext string. If you
+      prefer to reference your key using Databricks Secrets, see
+      `aws_secret_access_key`. You must provide an API key using one of the
+      following fields: `aws_secret_access_key` or
+      `aws_secret_access_key_plaintext`.
   "bedrock_provider":
     "description": |-
-      The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.
+      The underlying provider in Amazon Bedrock. Supported values (case
+      insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.
 github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfigBedrockProvider:
   "_":
-    "description": |-
-      The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.
     "enum":
       - |-
         anthropic
@@ -2812,10 +2840,16 @@ github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfigBedro
 github.com/databricks/databricks-sdk-go/service/serving.AnthropicConfig:
   "anthropic_api_key":
     "description": |-
-      The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.
+      The Databricks secret key reference for an Anthropic API key. If you
+      prefer to paste your API key directly, see `anthropic_api_key_plaintext`.
+      You must provide an API key using one of the following fields:
+      `anthropic_api_key` or `anthropic_api_key_plaintext`.
   "anthropic_api_key_plaintext":
     "description": |-
-      The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.
+      The Anthropic API key provided as a plaintext string. If you prefer to
+      reference your key using Databricks Secrets, see `anthropic_api_key`. You
+      must provide an API key using one of the following fields:
+      `anthropic_api_key` or `anthropic_api_key_plaintext`.
 github.com/databricks/databricks-sdk-go/service/serving.AutoCaptureConfigInput:
   "catalog_name":
     "description": |-
@@ -2831,42 +2865,58 @@ github.com/databricks/databricks-sdk-go/service/serving.AutoCaptureConfigInput:
       The prefix of the table in Unity Catalog. NOTE: On update, you cannot change the prefix name if the inference table is already enabled.
 github.com/databricks/databricks-sdk-go/service/serving.CohereConfig:
   "cohere_api_base":
-    "description": "This is an optional field to provide a customized base URL for the Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n"
+    "description": |-
+      This is an optional field to provide a customized base URL for the Cohere
+      API. If left unspecified, the standard Cohere base URL is used.
   "cohere_api_key":
     "description": |-
-      The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.
+      The Databricks secret key reference for a Cohere API key. If you prefer
+      to paste your API key directly, see `cohere_api_key_plaintext`. You must
+      provide an API key using one of the following fields: `cohere_api_key` or
+      `cohere_api_key_plaintext`.
   "cohere_api_key_plaintext":
     "description": |-
-      The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.
+      The Cohere API key provided as a plaintext string. If you prefer to
+      reference your key using Databricks Secrets, see `cohere_api_key`. You
+      must provide an API key using one of the following fields:
+      `cohere_api_key` or `cohere_api_key_plaintext`.
 github.com/databricks/databricks-sdk-go/service/serving.DatabricksModelServingConfig:
   "databricks_api_token":
-    "description": |
-      The Databricks secret key reference for a Databricks API token that corresponds to a user or service
-      principal with Can Query access to the model serving endpoint pointed to by this external model.
-      If you prefer to paste your API key directly, see `databricks_api_token_plaintext`.
-      You must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.
+    "description": |-
+      The Databricks secret key reference for a Databricks API token that
+      corresponds to a user or service principal with Can Query access to the
+      model serving endpoint pointed to by this external model. If you prefer
+      to paste your API key directly, see `databricks_api_token_plaintext`. You
+      must provide an API key using one of the following fields:
+      `databricks_api_token` or `databricks_api_token_plaintext`.
   "databricks_api_token_plaintext":
-    "description": |
-      The Databricks API token that corresponds to a user or service
-      principal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string.
-      If you prefer to reference your key using Databricks Secrets, see `databricks_api_token`.
-      You must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.
+    "description": |-
+      The Databricks API token that corresponds to a user or service principal
+      with Can Query access to the model serving endpoint pointed to by this
+      external model provided as a plaintext string. If you prefer to reference
+      your key using Databricks Secrets, see `databricks_api_token`. You must
+      provide an API key using one of the following fields:
+      `databricks_api_token` or `databricks_api_token_plaintext`.
   "databricks_workspace_url":
-    "description": |
-      The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model.
+    "description": |-
+      The URL of the Databricks workspace containing the model serving endpoint
+      pointed to by this external model.
 github.com/databricks/databricks-sdk-go/service/serving.EndpointCoreConfigInput:
   "auto_capture_config":
     "description": |-
       Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog.
+      Note: this field is deprecated for creating new provisioned throughput endpoints,
+      or updating existing provisioned throughput endpoints that never have inference table configured;
+      in these cases please use AI Gateway to manage inference tables.
   "served_entities":
     "description": |-
-      A list of served entities for the endpoint to serve. A serving endpoint can have up to 15 served entities.
+      The list of served entities under the serving endpoint config.
   "served_models":
     "description": |-
-      (Deprecated, use served_entities instead) A list of served models for the endpoint to serve. A serving endpoint can have up to 15 served models.
+      (Deprecated, use served_entities instead) The list of served models under the serving endpoint config.
   "traffic_config":
     "description": |-
-      The traffic config defining how invocations to the serving endpoint should be routed.
+      The traffic configuration associated with the serving endpoint config.
 github.com/databricks/databricks-sdk-go/service/serving.EndpointTag:
   "key":
     "description": |-
@@ -2903,17 +2953,13 @@ github.com/databricks/databricks-sdk-go/service/serving.ExternalModel:
     "description": |-
       PaLM Config. Only required if the provider is 'palm'.
   "provider":
-    "description": |
-      The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',
-      'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.",
+    "description": |-
+      The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic', 'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.
   "task":
     "description": |-
       The task type of the external model.
 github.com/databricks/databricks-sdk-go/service/serving.ExternalModelProvider:
   "_":
-    "description": |
-      The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',
-      'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.",
     "enum":
       - |-
         ai21labs
@@ -2934,70 +2980,114 @@ github.com/databricks/databricks-sdk-go/service/serving.ExternalModelProvider:
 github.com/databricks/databricks-sdk-go/service/serving.GoogleCloudVertexAiConfig:
   "private_key":
     "description": |-
-      The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`
+      The Databricks secret key reference for a private key for the service
+      account which has access to the Google Cloud Vertex AI Service. See [Best
+      practices for managing service account keys]. If you prefer to paste your
+      API key directly, see `private_key_plaintext`. You must provide an API
+      key using one of the following fields: `private_key` or
+      `private_key_plaintext`
+
+      [Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys
   "private_key_plaintext":
     "description": |-
-      The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`.
+      The private key for the service account which has access to the Google
+      Cloud Vertex AI Service provided as a plaintext secret. See [Best
+      practices for managing service account keys]. If you prefer to reference
+      your key using Databricks Secrets, see `private_key`. You must provide an
+      API key using one of the following fields: `private_key` or
+      `private_key_plaintext`.
+
+      [Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys
   "project_id":
     "description": |-
-      This is the Google Cloud project id that the service account is associated with.
+      This is the Google Cloud project id that the service account is
+      associated with.
   "region":
     "description": |-
-      This is the region for the Google Cloud Vertex AI Service. See [supported regions](https://cloud.google.com/vertex-ai/docs/general/locations) for more details. Some models are only available in specific regions.
+      This is the region for the Google Cloud Vertex AI Service. See [supported
+      regions] for more details. Some models are only available in specific
+      regions.
+
+      [supported regions]: https://cloud.google.com/vertex-ai/docs/general/locations
 github.com/databricks/databricks-sdk-go/service/serving.OpenAiConfig:
+  "_":
+    "description": |-
+      Configs needed to create an OpenAI model route.
   "microsoft_entra_client_id":
-    "description": |
-      This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.
+    "description": |-
+      This field is only required for Azure AD OpenAI and is the Microsoft
+      Entra Client ID.
   "microsoft_entra_client_secret":
-    "description": |
-      The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication.
-      If you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`.
-      You must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.
+    "description": |-
+      The Databricks secret key reference for a client secret used for
+      Microsoft Entra ID authentication. If you prefer to paste your client
+      secret directly, see `microsoft_entra_client_secret_plaintext`. You must
+      provide an API key using one of the following fields:
+      `microsoft_entra_client_secret` or
+      `microsoft_entra_client_secret_plaintext`.
   "microsoft_entra_client_secret_plaintext":
-    "description": |
-      The client secret used for Microsoft Entra ID authentication provided as a plaintext string.
-      If you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`.
-      You must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.
+    "description": |-
+      The client secret used for Microsoft Entra ID authentication provided as
+      a plaintext string. If you prefer to reference your key using Databricks
+      Secrets, see `microsoft_entra_client_secret`. You must provide an API key
+      using one of the following fields: `microsoft_entra_client_secret` or
+      `microsoft_entra_client_secret_plaintext`.
   "microsoft_entra_tenant_id":
-    "description": |
-      This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.
+    "description": |-
+      This field is only required for Azure AD OpenAI and is the Microsoft
+      Entra Tenant ID.
   "openai_api_base":
-    "description": |
-      This is a field to provide a customized base URl for the OpenAI API.
-      For Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service
-      provided by Azure.
-      For other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used.
+    "description": |-
+      This is a field to provide a customized base URl for the OpenAI API. For
+      Azure OpenAI, this field is required, and is the base URL for the Azure
+      OpenAI API service provided by Azure. For other OpenAI API types, this
+      field is optional, and if left unspecified, the standard OpenAI base URL
+      is used.
   "openai_api_key":
     "description": |-
-      The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`.
+      The Databricks secret key reference for an OpenAI API key using the
+      OpenAI or Azure service. If you prefer to paste your API key directly,
+      see `openai_api_key_plaintext`. You must provide an API key using one of
+      the following fields: `openai_api_key` or `openai_api_key_plaintext`.
   "openai_api_key_plaintext":
     "description": |-
-      The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`.
+      The OpenAI API key using the OpenAI or Azure service provided as a
+      plaintext string. If you prefer to reference your key using Databricks
+      Secrets, see `openai_api_key`. You must provide an API key using one of
+      the following fields: `openai_api_key` or `openai_api_key_plaintext`.
   "openai_api_type":
-    "description": |
-      This is an optional field to specify the type of OpenAI API to use.
-      For Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security
-      access validation protocol. For access token validation, use azure. For authentication using Azure Active
+    "description": |-
+      This is an optional field to specify the type of OpenAI API to use. For
+      Azure OpenAI, this field is required, and adjust this parameter to
+      represent the preferred security access validation protocol. For access
+      token validation, use azure. For authentication using Azure Active
       Directory (Azure AD) use, azuread.
   "openai_api_version":
-    "description": |
-      This is an optional field to specify the OpenAI API version.
-      For Azure OpenAI, this field is required, and is the version of the Azure OpenAI service to
-      utilize, specified by a date.
+    "description": |-
+      This is an optional field to specify the OpenAI API version. For Azure
+      OpenAI, this field is required, and is the version of the Azure OpenAI
+      service to utilize, specified by a date.
   "openai_deployment_name":
-    "description": |
-      This field is only required for Azure OpenAI and is the name of the deployment resource for the
-      Azure OpenAI service.
+    "description": |-
+      This field is only required for Azure OpenAI and is the name of the
+      deployment resource for the Azure OpenAI service.
   "openai_organization":
-    "description": |
-      This is an optional field to specify the organization in OpenAI or Azure OpenAI.
+    "description": |-
+      This is an optional field to specify the organization in OpenAI or Azure
+      OpenAI.
 github.com/databricks/databricks-sdk-go/service/serving.PaLmConfig:
   "palm_api_key":
     "description": |-
-      The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`.
+      The Databricks secret key reference for a PaLM API key. If you prefer to
+      paste your API key directly, see `palm_api_key_plaintext`. You must
+      provide an API key using one of the following fields: `palm_api_key` or
+      `palm_api_key_plaintext`.
   "palm_api_key_plaintext":
     "description": |-
-      The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`.
+      The PaLM API key provided as a plaintext string. If you prefer to
+      reference your key using Databricks Secrets, see `palm_api_key`. You must
+      provide an API key using one of the following fields: `palm_api_key` or
+      `palm_api_key_plaintext`.
 github.com/databricks/databricks-sdk-go/service/serving.RateLimit:
   "calls":
     "description": |-
@@ -3010,8 +3100,6 @@ github.com/databricks/databricks-sdk-go/service/serving.RateLimit:
       Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported.
 github.com/databricks/databricks-sdk-go/service/serving.RateLimitKey:
   "_":
-    "description": |-
-      Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.
     "enum":
       - |-
         user
@@ -3019,8 +3107,6 @@ github.com/databricks/databricks-sdk-go/service/serving.RateLimitKey:
         endpoint
 github.com/databricks/databricks-sdk-go/service/serving.RateLimitRenewalPeriod:
   "_":
-    "description": |-
-      Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported.
     "enum":
       - |-
         minute
@@ -3033,21 +3119,15 @@ github.com/databricks/databricks-sdk-go/service/serving.Route:
       The percentage of endpoint traffic to send to this route. It must be an integer between 0 and 100 inclusive.
 github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput:
   "entity_name":
-    "description": |
-      The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC),
-      or a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of
-      __catalog_name__.__schema_name__.__model_name__.
-  "entity_version":
     "description": |-
-      The version of the model in Databricks Model Registry to be served or empty if the entity is a FEATURE_SPEC.
+      The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC), or a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of **catalog_name.schema_name.model_name**.
+  "entity_version": {}
   "environment_vars":
-    "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity.\nNote: this is an experimental feature and subject to change. \nExample entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`"
+    "description": |-
+      An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{"OPENAI_API_KEY": "{{secrets/my_scope/my_key}}", "DATABRICKS_TOKEN": "{{secrets/my_scope2/my_key2}}"}`
   "external_model":
-    "description": |
-      The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)
-      can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model,
-      it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later.
-      The task type of all external models within an endpoint must be the same.
+    "description": |-
+      The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same.
   "instance_profile_arn":
     "description": |-
       ARN of the instance profile that the served entity uses to access AWS resources.
@@ -3058,68 +3138,46 @@ github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput:
     "description": |-
       The minimum tokens per second that the endpoint can scale down to.
   "name":
-    "description": |
-      The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores.
-      If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other
-      entities, it defaults to <entity-name>-<entity-version>.
+    "description": |-
+      The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.
   "scale_to_zero_enabled":
     "description": |-
       Whether the compute resources for the served entity should scale down to zero.
   "workload_size":
-    "description": |
-      The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between.
-      A single unit of provisioned concurrency can process one request at a time.
-      Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency).
-      If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.
+    "description": |-
+      The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.
   "workload_type":
-    "description": |
-      The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is
-      "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.
-      See the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).
+    "description": |-
+      The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).
 github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput:
   "environment_vars":
-    "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this model.\nNote: this is an experimental feature and subject to change. \nExample model environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`"
+    "description": |-
+      An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{"OPENAI_API_KEY": "{{secrets/my_scope/my_key}}", "DATABRICKS_TOKEN": "{{secrets/my_scope2/my_key2}}"}`
   "instance_profile_arn":
     "description": |-
-      ARN of the instance profile that the served model will use to access AWS resources.
+      ARN of the instance profile that the served entity uses to access AWS resources.
   "max_provisioned_throughput":
     "description": |-
       The maximum tokens per second that the endpoint can scale up to.
   "min_provisioned_throughput":
     "description": |-
       The minimum tokens per second that the endpoint can scale down to.
-  "model_name":
-    "description": |
-      The name of the model in Databricks Model Registry to be served or if the model resides in Unity Catalog, the full name of model,
-      in the form of __catalog_name__.__schema_name__.__model_name__.
-  "model_version":
-    "description": |-
-      The version of the model in Databricks Model Registry or Unity Catalog to be served.
+  "model_name": {}
+  "model_version": {}
   "name":
-    "description": |
-      The name of a served model. It must be unique across an endpoint. If not specified, this field will default to <model-name>-<model-version>.
-      A served model name can consist of alphanumeric characters, dashes, and underscores.
+    "description": |-
+      The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.
   "scale_to_zero_enabled":
     "description": |-
-      Whether the compute resources for the served model should scale down to zero.
+      Whether the compute resources for the served entity should scale down to zero.
   "workload_size":
-    "description": |
-      The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.
-      A single unit of provisioned concurrency can process one request at a time.
-      Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency).
-      If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.
+    "description": |-
+      The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.
   "workload_type":
-    "description": |
-      The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is
-      "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.
-      See the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).
+    "description": |-
+      The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).
 github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadSize:
   "_":
-    "description": |
-      The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.
-      A single unit of provisioned concurrency can process one request at a time.
-      Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency).
-      If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.
     "enum":
       - |-
         Small
@@ -3129,17 +3187,26 @@ github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkload
         Large
 github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadType:
   "_":
-    "description": |
-      The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is
-      "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.
-      See the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).
     "enum":
       - |-
         CPU
+      - |-
+        GPU_MEDIUM
       - |-
         GPU_SMALL
+      - |-
+        GPU_LARGE
+      - |-
+        MULTIGPU_MEDIUM
+github.com/databricks/databricks-sdk-go/service/serving.ServingModelWorkloadType:
+  "_":
+    "enum":
+      - |-
+        CPU
       - |-
         GPU_MEDIUM
+      - |-
+        GPU_SMALL
       - |-
         GPU_LARGE
       - |-
diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml
index 120a12543..323432fa3 100644
--- a/bundle/internal/schema/annotations_openapi_overrides.yml
+++ b/bundle/internal/schema/annotations_openapi_overrides.yml
@@ -197,3 +197,14 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger:
   "manual":
     "description": |-
       PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput:
+  "entity_version":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput:
+  "model_name":
+    "description": |-
+      PLACEHOLDER
+  "model_version":
+    "description": |-
+      PLACEHOLDER
diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json
index 4a3b56814..17a621ba0 100644
--- a/bundle/schema/jsonschema.json
+++ b/bundle/schema/jsonschema.json
@@ -546,7 +546,7 @@
                     "type": "object",
                     "properties": {
                       "ai_gateway": {
-                        "description": "The AI Gateway configuration for the serving endpoint. NOTE: only external model endpoints are supported as of now.",
+                        "description": "The AI Gateway configuration for the serving endpoint. NOTE: Only external model and provisioned throughput endpoints are currently supported.",
                         "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayConfig"
                       },
                       "config": {
@@ -554,7 +554,7 @@
                         "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.EndpointCoreConfigInput"
                       },
                       "name": {
-                        "description": "The name of the serving endpoint. This field is required and must be unique across a Databricks workspace.\nAn endpoint name can consist of alphanumeric characters, dashes, and underscores.\n",
+                        "description": "The name of the serving endpoint. This field is required and must be unique across a Databricks workspace.\nAn endpoint name can consist of alphanumeric characters, dashes, and underscores.",
                         "$ref": "#/$defs/string"
                       },
                       "permissions": {
@@ -575,7 +575,6 @@
                     },
                     "additionalProperties": false,
                     "required": [
-                      "config",
                       "name"
                     ]
                   },
@@ -4142,6 +4141,10 @@
                     "parameters": {
                       "description": "Parameters passed to the main method.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.",
                       "$ref": "#/$defs/slice/string"
+                    },
+                    "run_as_repl": {
+                      "description": "Deprecated. A value of `false` is no longer supported.",
+                      "$ref": "#/$defs/bool"
                     }
                   },
                   "additionalProperties": false
@@ -5502,11 +5505,11 @@
                   "type": "object",
                   "properties": {
                     "ai21labs_api_key": {
-                      "description": "The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`.",
+                      "description": "The Databricks secret key reference for an AI21 Labs API key. If you\nprefer to paste your API key directly, see `ai21labs_api_key_plaintext`.\nYou must provide an API key using one of the following fields:\n`ai21labs_api_key` or `ai21labs_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "ai21labs_api_key_plaintext": {
-                      "description": "An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`.",
+                      "description": "An AI21 Labs API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `ai21labs_api_key`. You\nmust provide an API key using one of the following fields:\n`ai21labs_api_key` or `ai21labs_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -5528,7 +5531,7 @@
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrails"
                     },
                     "inference_table_config": {
-                      "description": "Configuration for payload logging using inference tables. Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.",
+                      "description": "Configuration for payload logging using inference tables.\nUse these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayInferenceTableConfig"
                     },
                     "rate_limits": {
@@ -5536,7 +5539,7 @@
                       "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit"
                     },
                     "usage_tracking_config": {
-                      "description": "Configuration to enable usage tracking using system tables. These tables allow you to monitor operational usage on endpoints and their associated costs.",
+                      "description": "Configuration to enable usage tracking using system tables.\nThese tables allow you to monitor operational usage on endpoints and their associated costs.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayUsageTrackingConfig"
                     }
                   },
@@ -5554,7 +5557,7 @@
                   "type": "object",
                   "properties": {
                     "invalid_keywords": {
-                      "description": "List of invalid keywords. AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.",
+                      "description": "List of invalid keywords.\nAI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.",
                       "$ref": "#/$defs/slice/string"
                     },
                     "pii": {
@@ -5566,7 +5569,7 @@
                       "$ref": "#/$defs/bool"
                     },
                     "valid_topics": {
-                      "description": "The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics.",
+                      "description": "The list of allowed topics.\nGiven a chat request, this guardrail flags the request if its topic is not in the allowed topics.",
                       "$ref": "#/$defs/slice/string"
                     }
                   },
@@ -5584,14 +5587,11 @@
                   "type": "object",
                   "properties": {
                     "behavior": {
-                      "description": "Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.",
+                      "description": "Configuration for input guardrail filters.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehaviorBehavior"
                     }
                   },
-                  "additionalProperties": false,
-                  "required": [
-                    "behavior"
-                  ]
+                  "additionalProperties": false
                 },
                 {
                   "type": "string",
@@ -5603,7 +5603,6 @@
               "oneOf": [
                 {
                   "type": "string",
-                  "description": "Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.",
                   "enum": [
                     "NONE",
                     "BLOCK"
@@ -5643,7 +5642,7 @@
                   "type": "object",
                   "properties": {
                     "catalog_name": {
-                      "description": "The name of the catalog in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the catalog name.",
+                      "description": "The name of the catalog in Unity Catalog. Required when enabling inference tables.\nNOTE: On update, you have to disable inference table first in order to change the catalog name.",
                       "$ref": "#/$defs/string"
                     },
                     "enabled": {
@@ -5651,11 +5650,11 @@
                       "$ref": "#/$defs/bool"
                     },
                     "schema_name": {
-                      "description": "The name of the schema in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the schema name.",
+                      "description": "The name of the schema in Unity Catalog. Required when enabling inference tables.\nNOTE: On update, you have to disable inference table first in order to change the schema name.",
                       "$ref": "#/$defs/string"
                     },
                     "table_name_prefix": {
-                      "description": "The prefix of the table in Unity Catalog. NOTE: On update, you have to disable inference table first in order to change the prefix name.",
+                      "description": "The prefix of the table in Unity Catalog.\nNOTE: On update, you have to disable inference table first in order to change the prefix name.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -5674,10 +5673,10 @@
                   "properties": {
                     "calls": {
                       "description": "Used to specify how many calls are allowed for a key within the renewal_period.",
-                      "$ref": "#/$defs/int"
+                      "$ref": "#/$defs/int64"
                     },
                     "key": {
-                      "description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.",
+                      "description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported,\nwith 'endpoint' being the default if not specified.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitKey"
                     },
                     "renewal_period": {
@@ -5701,7 +5700,6 @@
               "oneOf": [
                 {
                   "type": "string",
-                  "description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.",
                   "enum": [
                     "user",
                     "endpoint"
@@ -5717,7 +5715,6 @@
               "oneOf": [
                 {
                   "type": "string",
-                  "description": "Renewal period field for a rate limit. Currently, only 'minute' is supported.",
                   "enum": [
                     "minute"
                   ]
@@ -5752,11 +5749,11 @@
                   "type": "object",
                   "properties": {
                     "aws_access_key_id": {
-                      "description": "The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.",
+                      "description": "The Databricks secret key reference for an AWS access key ID with\npermissions to interact with Bedrock services. If you prefer to paste\nyour API key directly, see `aws_access_key_id_plaintext`. You must provide an API\nkey using one of the following fields: `aws_access_key_id` or\n`aws_access_key_id_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "aws_access_key_id_plaintext": {
-                      "description": "An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.",
+                      "description": "An AWS access key ID with permissions to interact with Bedrock services\nprovided as a plaintext string. If you prefer to reference your key using\nDatabricks Secrets, see `aws_access_key_id`. You must provide an API key\nusing one of the following fields: `aws_access_key_id` or\n`aws_access_key_id_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "aws_region": {
@@ -5764,15 +5761,15 @@
                       "$ref": "#/$defs/string"
                     },
                     "aws_secret_access_key": {
-                      "description": "The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.",
+                      "description": "The Databricks secret key reference for an AWS secret access key paired\nwith the access key ID, with permissions to interact with Bedrock\nservices. If you prefer to paste your API key directly, see\n`aws_secret_access_key_plaintext`. You must provide an API key using one\nof the following fields: `aws_secret_access_key` or\n`aws_secret_access_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "aws_secret_access_key_plaintext": {
-                      "description": "An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.",
+                      "description": "An AWS secret access key paired with the access key ID, with permissions\nto interact with Bedrock services provided as a plaintext string. If you\nprefer to reference your key using Databricks Secrets, see\n`aws_secret_access_key`. You must provide an API key using one of the\nfollowing fields: `aws_secret_access_key` or\n`aws_secret_access_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "bedrock_provider": {
-                      "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.",
+                      "description": "The underlying provider in Amazon Bedrock. Supported values (case\ninsensitive) include: Anthropic, Cohere, AI21Labs, Amazon.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfigBedrockProvider"
                     }
                   },
@@ -5792,7 +5789,6 @@
               "oneOf": [
                 {
                   "type": "string",
-                  "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.",
                   "enum": [
                     "anthropic",
                     "cohere",
@@ -5812,11 +5808,11 @@
                   "type": "object",
                   "properties": {
                     "anthropic_api_key": {
-                      "description": "The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.",
+                      "description": "The Databricks secret key reference for an Anthropic API key. If you\nprefer to paste your API key directly, see `anthropic_api_key_plaintext`.\nYou must provide an API key using one of the following fields:\n`anthropic_api_key` or `anthropic_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "anthropic_api_key_plaintext": {
-                      "description": "The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.",
+                      "description": "The Anthropic API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `anthropic_api_key`. You\nmust provide an API key using one of the following fields:\n`anthropic_api_key` or `anthropic_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -5864,15 +5860,15 @@
                   "type": "object",
                   "properties": {
                     "cohere_api_base": {
-                      "description": "This is an optional field to provide a customized base URL for the Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n",
+                      "description": "This is an optional field to provide a customized base URL for the Cohere\nAPI. If left unspecified, the standard Cohere base URL is used.",
                       "$ref": "#/$defs/string"
                     },
                     "cohere_api_key": {
-                      "description": "The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.",
+                      "description": "The Databricks secret key reference for a Cohere API key. If you prefer\nto paste your API key directly, see `cohere_api_key_plaintext`. You must\nprovide an API key using one of the following fields: `cohere_api_key` or\n`cohere_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "cohere_api_key_plaintext": {
-                      "description": "The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.",
+                      "description": "The Cohere API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `cohere_api_key`. You\nmust provide an API key using one of the following fields:\n`cohere_api_key` or `cohere_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -5890,15 +5886,15 @@
                   "type": "object",
                   "properties": {
                     "databricks_api_token": {
-                      "description": "The Databricks secret key reference for a Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model.\nIf you prefer to paste your API key directly, see `databricks_api_token_plaintext`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n",
+                      "description": "The Databricks secret key reference for a Databricks API token that\ncorresponds to a user or service principal with Can Query access to the\nmodel serving endpoint pointed to by this external model. If you prefer\nto paste your API key directly, see `databricks_api_token_plaintext`. You\nmust provide an API key using one of the following fields:\n`databricks_api_token` or `databricks_api_token_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "databricks_api_token_plaintext": {
-                      "description": "The Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `databricks_api_token`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n",
+                      "description": "The Databricks API token that corresponds to a user or service principal\nwith Can Query access to the model serving endpoint pointed to by this\nexternal model provided as a plaintext string. If you prefer to reference\nyour key using Databricks Secrets, see `databricks_api_token`. You must\nprovide an API key using one of the following fields:\n`databricks_api_token` or `databricks_api_token_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "databricks_workspace_url": {
-                      "description": "The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model.\n",
+                      "description": "The URL of the Databricks workspace containing the model serving endpoint\npointed to by this external model.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -5919,19 +5915,19 @@
                   "type": "object",
                   "properties": {
                     "auto_capture_config": {
-                      "description": "Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog.",
+                      "description": "Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog.\nNote: this field is deprecated for creating new provisioned throughput endpoints,\nor updating existing provisioned throughput endpoints that never have inference table configured;\nin these cases please use AI Gateway to manage inference tables.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AutoCaptureConfigInput"
                     },
                     "served_entities": {
-                      "description": "A list of served entities for the endpoint to serve. A serving endpoint can have up to 15 served entities.",
+                      "description": "The list of served entities under the serving endpoint config.",
                       "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput"
                     },
                     "served_models": {
-                      "description": "(Deprecated, use served_entities instead) A list of served models for the endpoint to serve. A serving endpoint can have up to 15 served models.",
+                      "description": "(Deprecated, use served_entities instead) The list of served models under the serving endpoint config.",
                       "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput"
                     },
                     "traffic_config": {
-                      "description": "The traffic config defining how invocations to the serving endpoint should be routed.",
+                      "description": "The traffic configuration associated with the serving endpoint config.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.TrafficConfig"
                     }
                   },
@@ -6010,7 +6006,7 @@
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.PaLmConfig"
                     },
                     "provider": {
-                      "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n",
+                      "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic', 'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ExternalModelProvider"
                     },
                     "task": {
@@ -6035,7 +6031,6 @@
               "oneOf": [
                 {
                   "type": "string",
-                  "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n",
                   "enum": [
                     "ai21labs",
                     "anthropic",
@@ -6059,23 +6054,27 @@
                   "type": "object",
                   "properties": {
                     "private_key": {
-                      "description": "The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`",
+                      "description": "The Databricks secret key reference for a private key for the service\naccount which has access to the Google Cloud Vertex AI Service. See [Best\npractices for managing service account keys]. If you prefer to paste your\nAPI key directly, see `private_key_plaintext`. You must provide an API\nkey using one of the following fields: `private_key` or\n`private_key_plaintext`\n\n[Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys",
                       "$ref": "#/$defs/string"
                     },
                     "private_key_plaintext": {
-                      "description": "The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`.",
+                      "description": "The private key for the service account which has access to the Google\nCloud Vertex AI Service provided as a plaintext secret. See [Best\npractices for managing service account keys]. If you prefer to reference\nyour key using Databricks Secrets, see `private_key`. You must provide an\nAPI key using one of the following fields: `private_key` or\n`private_key_plaintext`.\n\n[Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys",
                       "$ref": "#/$defs/string"
                     },
                     "project_id": {
-                      "description": "This is the Google Cloud project id that the service account is associated with.",
+                      "description": "This is the Google Cloud project id that the service account is\nassociated with.",
                       "$ref": "#/$defs/string"
                     },
                     "region": {
-                      "description": "This is the region for the Google Cloud Vertex AI Service. See [supported regions](https://cloud.google.com/vertex-ai/docs/general/locations) for more details. Some models are only available in specific regions.",
+                      "description": "This is the region for the Google Cloud Vertex AI Service. See [supported\nregions] for more details. Some models are only available in specific\nregions.\n\n[supported regions]: https://cloud.google.com/vertex-ai/docs/general/locations",
                       "$ref": "#/$defs/string"
                     }
                   },
-                  "additionalProperties": false
+                  "additionalProperties": false,
+                  "required": [
+                    "project_id",
+                    "region"
+                  ]
                 },
                 {
                   "type": "string",
@@ -6087,49 +6086,50 @@
               "oneOf": [
                 {
                   "type": "object",
+                  "description": "Configs needed to create an OpenAI model route.",
                   "properties": {
                     "microsoft_entra_client_id": {
-                      "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.\n",
+                      "description": "This field is only required for Azure AD OpenAI and is the Microsoft\nEntra Client ID.",
                       "$ref": "#/$defs/string"
                     },
                     "microsoft_entra_client_secret": {
-                      "description": "The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication.\nIf you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n",
+                      "description": "The Databricks secret key reference for a client secret used for\nMicrosoft Entra ID authentication. If you prefer to paste your client\nsecret directly, see `microsoft_entra_client_secret_plaintext`. You must\nprovide an API key using one of the following fields:\n`microsoft_entra_client_secret` or\n`microsoft_entra_client_secret_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "microsoft_entra_client_secret_plaintext": {
-                      "description": "The client secret used for Microsoft Entra ID authentication provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n",
+                      "description": "The client secret used for Microsoft Entra ID authentication provided as\na plaintext string. If you prefer to reference your key using Databricks\nSecrets, see `microsoft_entra_client_secret`. You must provide an API key\nusing one of the following fields: `microsoft_entra_client_secret` or\n`microsoft_entra_client_secret_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "microsoft_entra_tenant_id": {
-                      "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.\n",
+                      "description": "This field is only required for Azure AD OpenAI and is the Microsoft\nEntra Tenant ID.",
                       "$ref": "#/$defs/string"
                     },
                     "openai_api_base": {
-                      "description": "This is a field to provide a customized base URl for the OpenAI API.\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\nFor other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used.\n",
+                      "description": "This is a field to provide a customized base URl for the OpenAI API. For\nAzure OpenAI, this field is required, and is the base URL for the Azure\nOpenAI API service provided by Azure. For other OpenAI API types, this\nfield is optional, and if left unspecified, the standard OpenAI base URL\nis used.",
                       "$ref": "#/$defs/string"
                     },
                     "openai_api_key": {
-                      "description": "The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`.",
+                      "description": "The Databricks secret key reference for an OpenAI API key using the\nOpenAI or Azure service. If you prefer to paste your API key directly,\nsee `openai_api_key_plaintext`. You must provide an API key using one of\nthe following fields: `openai_api_key` or `openai_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "openai_api_key_plaintext": {
-                      "description": "The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`.",
+                      "description": "The OpenAI API key using the OpenAI or Azure service provided as a\nplaintext string. If you prefer to reference your key using Databricks\nSecrets, see `openai_api_key`. You must provide an API key using one of\nthe following fields: `openai_api_key` or `openai_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "openai_api_type": {
-                      "description": "This is an optional field to specify the type of OpenAI API to use.\nFor Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security\naccess validation protocol. For access token validation, use azure. For authentication using Azure Active\nDirectory (Azure AD) use, azuread.\n",
+                      "description": "This is an optional field to specify the type of OpenAI API to use. For\nAzure OpenAI, this field is required, and adjust this parameter to\nrepresent the preferred security access validation protocol. For access\ntoken validation, use azure. For authentication using Azure Active\nDirectory (Azure AD) use, azuread.",
                       "$ref": "#/$defs/string"
                     },
                     "openai_api_version": {
-                      "description": "This is an optional field to specify the OpenAI API version.\nFor Azure OpenAI, this field is required, and is the version of the Azure OpenAI service to\nutilize, specified by a date.\n",
+                      "description": "This is an optional field to specify the OpenAI API version. For Azure\nOpenAI, this field is required, and is the version of the Azure OpenAI\nservice to utilize, specified by a date.",
                       "$ref": "#/$defs/string"
                     },
                     "openai_deployment_name": {
-                      "description": "This field is only required for Azure OpenAI and is the name of the deployment resource for the\nAzure OpenAI service.\n",
+                      "description": "This field is only required for Azure OpenAI and is the name of the\ndeployment resource for the Azure OpenAI service.",
                       "$ref": "#/$defs/string"
                     },
                     "openai_organization": {
-                      "description": "This is an optional field to specify the organization in OpenAI or Azure OpenAI.\n",
+                      "description": "This is an optional field to specify the organization in OpenAI or Azure\nOpenAI.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -6147,11 +6147,11 @@
                   "type": "object",
                   "properties": {
                     "palm_api_key": {
-                      "description": "The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`.",
+                      "description": "The Databricks secret key reference for a PaLM API key. If you prefer to\npaste your API key directly, see `palm_api_key_plaintext`. You must\nprovide an API key using one of the following fields: `palm_api_key` or\n`palm_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     },
                     "palm_api_key_plaintext": {
-                      "description": "The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`.",
+                      "description": "The PaLM API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `palm_api_key`. You must\nprovide an API key using one of the following fields: `palm_api_key` or\n`palm_api_key_plaintext`.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -6170,7 +6170,7 @@
                   "properties": {
                     "calls": {
                       "description": "Used to specify how many calls are allowed for a key within the renewal_period.",
-                      "$ref": "#/$defs/int"
+                      "$ref": "#/$defs/int64"
                     },
                     "key": {
                       "description": "Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.",
@@ -6197,7 +6197,6 @@
               "oneOf": [
                 {
                   "type": "string",
-                  "description": "Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.",
                   "enum": [
                     "user",
                     "endpoint"
@@ -6213,7 +6212,6 @@
               "oneOf": [
                 {
                   "type": "string",
-                  "description": "Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported.",
                   "enum": [
                     "minute"
                   ]
@@ -6256,19 +6254,18 @@
                   "type": "object",
                   "properties": {
                     "entity_name": {
-                      "description": "The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC),\nor a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of\n__catalog_name__.__schema_name__.__model_name__.\n",
+                      "description": "The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC), or a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of **catalog_name.schema_name.model_name**.",
                       "$ref": "#/$defs/string"
                     },
                     "entity_version": {
-                      "description": "The version of the model in Databricks Model Registry to be served or empty if the entity is a FEATURE_SPEC.",
                       "$ref": "#/$defs/string"
                     },
                     "environment_vars": {
-                      "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity.\nNote: this is an experimental feature and subject to change. \nExample entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`",
+                      "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`",
                       "$ref": "#/$defs/map/string"
                     },
                     "external_model": {
-                      "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)\ncan be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model,\nit cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later.\nThe task type of all external models within an endpoint must be the same.\n",
+                      "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ExternalModel"
                     },
                     "instance_profile_arn": {
@@ -6284,7 +6281,7 @@
                       "$ref": "#/$defs/int"
                     },
                     "name": {
-                      "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores.\nIf not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other\nentities, it defaults to \u003centity-name\u003e-\u003centity-version\u003e.\n",
+                      "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.",
                       "$ref": "#/$defs/string"
                     },
                     "scale_to_zero_enabled": {
@@ -6292,12 +6289,12 @@
                       "$ref": "#/$defs/bool"
                     },
                     "workload_size": {
-                      "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.\n",
+                      "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.",
                       "$ref": "#/$defs/string"
                     },
                     "workload_type": {
-                      "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n",
-                      "$ref": "#/$defs/string"
+                      "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is \"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).",
+                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServingModelWorkloadType"
                     }
                   },
                   "additionalProperties": false
@@ -6314,11 +6311,11 @@
                   "type": "object",
                   "properties": {
                     "environment_vars": {
-                      "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this model.\nNote: this is an experimental feature and subject to change. \nExample model environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`",
+                      "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`",
                       "$ref": "#/$defs/map/string"
                     },
                     "instance_profile_arn": {
-                      "description": "ARN of the instance profile that the served model will use to access AWS resources.",
+                      "description": "ARN of the instance profile that the served entity uses to access AWS resources.",
                       "$ref": "#/$defs/string"
                     },
                     "max_provisioned_throughput": {
@@ -6330,27 +6327,25 @@
                       "$ref": "#/$defs/int"
                     },
                     "model_name": {
-                      "description": "The name of the model in Databricks Model Registry to be served or if the model resides in Unity Catalog, the full name of model,\nin the form of __catalog_name__.__schema_name__.__model_name__.\n",
                       "$ref": "#/$defs/string"
                     },
                     "model_version": {
-                      "description": "The version of the model in Databricks Model Registry or Unity Catalog to be served.",
                       "$ref": "#/$defs/string"
                     },
                     "name": {
-                      "description": "The name of a served model. It must be unique across an endpoint. If not specified, this field will default to \u003cmodel-name\u003e-\u003cmodel-version\u003e.\nA served model name can consist of alphanumeric characters, dashes, and underscores.\n",
+                      "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.",
                       "$ref": "#/$defs/string"
                     },
                     "scale_to_zero_enabled": {
-                      "description": "Whether the compute resources for the served model should scale down to zero.",
+                      "description": "Whether the compute resources for the served entity should scale down to zero.",
                       "$ref": "#/$defs/bool"
                     },
                     "workload_size": {
-                      "description": "The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.\n",
+                      "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadSize"
                     },
                     "workload_type": {
-                      "description": "The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n",
+                      "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is \"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadType"
                     }
                   },
@@ -6371,7 +6366,6 @@
               "oneOf": [
                 {
                   "type": "string",
-                  "description": "The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.\n",
                   "enum": [
                     "Small",
                     "Medium",
@@ -6388,11 +6382,28 @@
               "oneOf": [
                 {
                   "type": "string",
-                  "description": "The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n",
                   "enum": [
                     "CPU",
-                    "GPU_SMALL",
                     "GPU_MEDIUM",
+                    "GPU_SMALL",
+                    "GPU_LARGE",
+                    "MULTIGPU_MEDIUM"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
+              ]
+            },
+            "serving.ServingModelWorkloadType": {
+              "oneOf": [
+                {
+                  "type": "string",
+                  "enum": [
+                    "CPU",
+                    "GPU_MEDIUM",
+                    "GPU_SMALL",
                     "GPU_LARGE",
                     "MULTIGPU_MEDIUM"
                   ]
diff --git a/cmd/account/custom-app-integration/custom-app-integration.go b/cmd/account/custom-app-integration/custom-app-integration.go
index 1eec1018e..43e458bc6 100755
--- a/cmd/account/custom-app-integration/custom-app-integration.go
+++ b/cmd/account/custom-app-integration/custom-app-integration.go
@@ -307,6 +307,7 @@ func newUpdate() *cobra.Command {
 	cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
 	// TODO: array: redirect_urls
+	// TODO: array: scopes
 	// TODO: complex arg: token_access_policy
 
 	cmd.Use = "update INTEGRATION_ID"
diff --git a/cmd/api/api.go b/cmd/api/api.go
index c3a3eb0b6..fad8a026f 100644
--- a/cmd/api/api.go
+++ b/cmd/api/api.go
@@ -62,7 +62,7 @@ func makeCommand(method string) *cobra.Command {
 
 			var response any
 			headers := map[string]string{"Content-Type": "application/json"}
-			err = api.Do(cmd.Context(), method, path, headers, request, &response)
+			err = api.Do(cmd.Context(), method, path, headers, nil, request, &response)
 			if err != nil {
 				return err
 			}
diff --git a/cmd/workspace/access-control/access-control.go b/cmd/workspace/access-control/access-control.go
new file mode 100755
index 000000000..7668265fb
--- /dev/null
+++ b/cmd/workspace/access-control/access-control.go
@@ -0,0 +1,109 @@
+// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT.
+
+package access_control
+
+import (
+	"fmt"
+
+	"github.com/databricks/cli/cmd/root"
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/flags"
+	"github.com/databricks/databricks-sdk-go/service/iam"
+	"github.com/spf13/cobra"
+)
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var cmdOverrides []func(*cobra.Command)
+
+func New() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "access-control",
+		Short:   `Rule based Access Control for Databricks Resources.`,
+		Long:    `Rule based Access Control for Databricks Resources.`,
+		GroupID: "iam",
+		Annotations: map[string]string{
+			"package": "iam",
+		},
+
+		// This service is being previewed; hide from help output.
+		Hidden: true,
+	}
+
+	// Add methods
+	cmd.AddCommand(newCheckPolicy())
+
+	// Apply optional overrides to this command.
+	for _, fn := range cmdOverrides {
+		fn(cmd)
+	}
+
+	return cmd
+}
+
+// start check-policy command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var checkPolicyOverrides []func(
+	*cobra.Command,
+	*iam.CheckPolicyRequest,
+)
+
+func newCheckPolicy() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var checkPolicyReq iam.CheckPolicyRequest
+	var checkPolicyJson flags.JsonFlag
+
+	// TODO: short flags
+	cmd.Flags().Var(&checkPolicyJson, "json", `either inline JSON string or @path/to/file.json with request body`)
+
+	// TODO: complex arg: resource_info
+
+	cmd.Use = "check-policy"
+	cmd.Short = `Check access policy to a resource.`
+	cmd.Long = `Check access policy to a resource.`
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.PreRunE = root.MustWorkspaceClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		w := root.WorkspaceClient(ctx)
+
+		if cmd.Flags().Changed("json") {
+			diags := checkPolicyJson.Unmarshal(&checkPolicyReq)
+			if diags.HasError() {
+				return diags.Error()
+			}
+			if len(diags) > 0 {
+				err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags)
+				if err != nil {
+					return err
+				}
+			}
+		} else {
+			return fmt.Errorf("please provide command input in JSON format by specifying the --json flag")
+		}
+
+		response, err := w.AccessControl.CheckPolicy(ctx, checkPolicyReq)
+		if err != nil {
+			return err
+		}
+		return cmdio.Render(ctx, response)
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range checkPolicyOverrides {
+		fn(cmd, &checkPolicyReq)
+	}
+
+	return cmd
+}
+
+// end service AccessControl
diff --git a/cmd/workspace/cmd.go b/cmd/workspace/cmd.go
index f07d0cf76..c447bd736 100755
--- a/cmd/workspace/cmd.go
+++ b/cmd/workspace/cmd.go
@@ -3,6 +3,7 @@
 package workspace
 
 import (
+	access_control "github.com/databricks/cli/cmd/workspace/access-control"
 	alerts "github.com/databricks/cli/cmd/workspace/alerts"
 	alerts_legacy "github.com/databricks/cli/cmd/workspace/alerts-legacy"
 	apps "github.com/databricks/cli/cmd/workspace/apps"
@@ -96,6 +97,7 @@ import (
 func All() []*cobra.Command {
 	var out []*cobra.Command
 
+	out = append(out, access_control.New())
 	out = append(out, alerts.New())
 	out = append(out, alerts_legacy.New())
 	out = append(out, apps.New())
diff --git a/cmd/workspace/providers/providers.go b/cmd/workspace/providers/providers.go
index 504beac5e..4d6262cff 100755
--- a/cmd/workspace/providers/providers.go
+++ b/cmd/workspace/providers/providers.go
@@ -64,7 +64,7 @@ func newCreate() *cobra.Command {
 	cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
 	cmd.Flags().StringVar(&createReq.Comment, "comment", createReq.Comment, `Description about the provider.`)
-	cmd.Flags().StringVar(&createReq.RecipientProfileStr, "recipient-profile-str", createReq.RecipientProfileStr, `This field is required when the __authentication_type__ is **TOKEN** or not provided.`)
+	cmd.Flags().StringVar(&createReq.RecipientProfileStr, "recipient-profile-str", createReq.RecipientProfileStr, `This field is required when the __authentication_type__ is **TOKEN**, **OAUTH_CLIENT_CREDENTIALS** or not provided.`)
 
 	cmd.Use = "create NAME AUTHENTICATION_TYPE"
 	cmd.Short = `Create an auth provider.`
@@ -430,7 +430,7 @@ func newUpdate() *cobra.Command {
 	cmd.Flags().StringVar(&updateReq.Comment, "comment", updateReq.Comment, `Description about the provider.`)
 	cmd.Flags().StringVar(&updateReq.NewName, "new-name", updateReq.NewName, `New name for the provider.`)
 	cmd.Flags().StringVar(&updateReq.Owner, "owner", updateReq.Owner, `Username of Provider owner.`)
-	cmd.Flags().StringVar(&updateReq.RecipientProfileStr, "recipient-profile-str", updateReq.RecipientProfileStr, `This field is required when the __authentication_type__ is **TOKEN** or not provided.`)
+	cmd.Flags().StringVar(&updateReq.RecipientProfileStr, "recipient-profile-str", updateReq.RecipientProfileStr, `This field is required when the __authentication_type__ is **TOKEN**, **OAUTH_CLIENT_CREDENTIALS** or not provided.`)
 
 	cmd.Use = "update NAME"
 	cmd.Short = `Update a provider.`
diff --git a/cmd/workspace/recipients/recipients.go b/cmd/workspace/recipients/recipients.go
index 56abd2014..6d6ce42f1 100755
--- a/cmd/workspace/recipients/recipients.go
+++ b/cmd/workspace/recipients/recipients.go
@@ -91,7 +91,7 @@ func newCreate() *cobra.Command {
 	cmd.Long = `Create a share recipient.
   
   Creates a new recipient with the delta sharing authentication type in the
-  metastore. The caller must be a metastore admin or has the
+  metastore. The caller must be a metastore admin or have the
   **CREATE_RECIPIENT** privilege on the metastore.
 
   Arguments:
@@ -186,28 +186,16 @@ func newDelete() *cobra.Command {
 
 	cmd.Annotations = make(map[string]string)
 
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
 	cmd.PreRunE = root.MustWorkspaceClient
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
 
-		if len(args) == 0 {
-			promptSpinner := cmdio.Spinner(ctx)
-			promptSpinner <- "No NAME argument specified. Loading names for Recipients drop-down."
-			names, err := w.Recipients.RecipientInfoNameToMetastoreIdMap(ctx, sharing.ListRecipientsRequest{})
-			close(promptSpinner)
-			if err != nil {
-				return fmt.Errorf("failed to load names for Recipients drop-down. Please manually specify required arguments. Original error: %w", err)
-			}
-			id, err := cmdio.Select(ctx, names, "Name of the recipient")
-			if err != nil {
-				return err
-			}
-			args = append(args, id)
-		}
-		if len(args) != 1 {
-			return fmt.Errorf("expected to have name of the recipient")
-		}
 		deleteReq.Name = args[0]
 
 		err = w.Recipients.Delete(ctx, deleteReq)
@@ -258,28 +246,16 @@ func newGet() *cobra.Command {
 
 	cmd.Annotations = make(map[string]string)
 
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
 	cmd.PreRunE = root.MustWorkspaceClient
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
 
-		if len(args) == 0 {
-			promptSpinner := cmdio.Spinner(ctx)
-			promptSpinner <- "No NAME argument specified. Loading names for Recipients drop-down."
-			names, err := w.Recipients.RecipientInfoNameToMetastoreIdMap(ctx, sharing.ListRecipientsRequest{})
-			close(promptSpinner)
-			if err != nil {
-				return fmt.Errorf("failed to load names for Recipients drop-down. Please manually specify required arguments. Original error: %w", err)
-			}
-			id, err := cmdio.Select(ctx, names, "Name of the recipient")
-			if err != nil {
-				return err
-			}
-			args = append(args, id)
-		}
-		if len(args) != 1 {
-			return fmt.Errorf("expected to have name of the recipient")
-		}
 		getReq.Name = args[0]
 
 		response, err := w.Recipients.Get(ctx, getReq)
@@ -384,7 +360,7 @@ func newRotateToken() *cobra.Command {
   the provided token info. The caller must be the owner of the recipient.
 
   Arguments:
-    NAME: The name of the recipient.
+    NAME: The name of the Recipient.
     EXISTING_TOKEN_EXPIRE_IN_SECONDS: The expiration time of the bearer token in ISO 8601 format. This will set
       the expiration_time of existing token only to a smaller timestamp, it
       cannot extend the expiration_time. Use 0 to expire the existing token
@@ -479,28 +455,16 @@ func newSharePermissions() *cobra.Command {
 
 	cmd.Annotations = make(map[string]string)
 
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
 	cmd.PreRunE = root.MustWorkspaceClient
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
 
-		if len(args) == 0 {
-			promptSpinner := cmdio.Spinner(ctx)
-			promptSpinner <- "No NAME argument specified. Loading names for Recipients drop-down."
-			names, err := w.Recipients.RecipientInfoNameToMetastoreIdMap(ctx, sharing.ListRecipientsRequest{})
-			close(promptSpinner)
-			if err != nil {
-				return fmt.Errorf("failed to load names for Recipients drop-down. Please manually specify required arguments. Original error: %w", err)
-			}
-			id, err := cmdio.Select(ctx, names, "The name of the Recipient")
-			if err != nil {
-				return err
-			}
-			args = append(args, id)
-		}
-		if len(args) != 1 {
-			return fmt.Errorf("expected to have the name of the recipient")
-		}
 		sharePermissionsReq.Name = args[0]
 
 		response, err := w.Recipients.SharePermissions(ctx, sharePermissionsReq)
@@ -560,6 +524,11 @@ func newUpdate() *cobra.Command {
 
 	cmd.Annotations = make(map[string]string)
 
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
 	cmd.PreRunE = root.MustWorkspaceClient
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
@@ -577,30 +546,13 @@ func newUpdate() *cobra.Command {
 				}
 			}
 		}
-		if len(args) == 0 {
-			promptSpinner := cmdio.Spinner(ctx)
-			promptSpinner <- "No NAME argument specified. Loading names for Recipients drop-down."
-			names, err := w.Recipients.RecipientInfoNameToMetastoreIdMap(ctx, sharing.ListRecipientsRequest{})
-			close(promptSpinner)
-			if err != nil {
-				return fmt.Errorf("failed to load names for Recipients drop-down. Please manually specify required arguments. Original error: %w", err)
-			}
-			id, err := cmdio.Select(ctx, names, "Name of the recipient")
-			if err != nil {
-				return err
-			}
-			args = append(args, id)
-		}
-		if len(args) != 1 {
-			return fmt.Errorf("expected to have name of the recipient")
-		}
 		updateReq.Name = args[0]
 
-		err = w.Recipients.Update(ctx, updateReq)
+		response, err := w.Recipients.Update(ctx, updateReq)
 		if err != nil {
 			return err
 		}
-		return nil
+		return cmdio.Render(ctx, response)
 	}
 
 	// Disable completions since they are not applicable.
diff --git a/cmd/workspace/serving-endpoints/serving-endpoints.go b/cmd/workspace/serving-endpoints/serving-endpoints.go
index cc99177c7..034133623 100755
--- a/cmd/workspace/serving-endpoints/serving-endpoints.go
+++ b/cmd/workspace/serving-endpoints/serving-endpoints.go
@@ -49,6 +49,7 @@ func New() *cobra.Command {
 	cmd.AddCommand(newGetOpenApi())
 	cmd.AddCommand(newGetPermissionLevels())
 	cmd.AddCommand(newGetPermissions())
+	cmd.AddCommand(newHttpRequest())
 	cmd.AddCommand(newList())
 	cmd.AddCommand(newLogs())
 	cmd.AddCommand(newPatch())
@@ -153,16 +154,34 @@ func newCreate() *cobra.Command {
 	cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
 	// TODO: complex arg: ai_gateway
+	// TODO: complex arg: config
 	// TODO: array: rate_limits
 	cmd.Flags().BoolVar(&createReq.RouteOptimized, "route-optimized", createReq.RouteOptimized, `Enable route optimization for the serving endpoint.`)
 	// TODO: array: tags
 
-	cmd.Use = "create"
+	cmd.Use = "create NAME"
 	cmd.Short = `Create a new serving endpoint.`
-	cmd.Long = `Create a new serving endpoint.`
+	cmd.Long = `Create a new serving endpoint.
+
+  Arguments:
+    NAME: The name of the serving endpoint. This field is required and must be
+      unique across a Databricks workspace. An endpoint name can consist of
+      alphanumeric characters, dashes, and underscores.`
 
 	cmd.Annotations = make(map[string]string)
 
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		if cmd.Flags().Changed("json") {
+			err := root.ExactArgs(0)(cmd, args)
+			if err != nil {
+				return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'name' in your JSON input")
+			}
+			return nil
+		}
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
 	cmd.PreRunE = root.MustWorkspaceClient
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
@@ -179,8 +198,9 @@ func newCreate() *cobra.Command {
 					return err
 				}
 			}
-		} else {
-			return fmt.Errorf("please provide command input in JSON format by specifying the --json flag")
+		}
+		if !cmd.Flags().Changed("json") {
+			createReq.Name = args[0]
 		}
 
 		wait, err := w.ServingEndpoints.Create(ctx, createReq)
@@ -233,10 +253,7 @@ func newDelete() *cobra.Command {
 
 	cmd.Use = "delete NAME"
 	cmd.Short = `Delete a serving endpoint.`
-	cmd.Long = `Delete a serving endpoint.
-
-  Arguments:
-    NAME: The name of the serving endpoint. This field is required.`
+	cmd.Long = `Delete a serving endpoint.`
 
 	cmd.Annotations = make(map[string]string)
 
@@ -432,11 +449,12 @@ func newGetOpenApi() *cobra.Command {
 
 		getOpenApiReq.Name = args[0]
 
-		err = w.ServingEndpoints.GetOpenApi(ctx, getOpenApiReq)
+		response, err := w.ServingEndpoints.GetOpenApi(ctx, getOpenApiReq)
 		if err != nil {
 			return err
 		}
-		return nil
+		defer response.Contents.Close()
+		return cmdio.Render(ctx, response.Contents)
 	}
 
 	// Disable completions since they are not applicable.
@@ -568,6 +586,77 @@ func newGetPermissions() *cobra.Command {
 	return cmd
 }
 
+// start http-request command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var httpRequestOverrides []func(
+	*cobra.Command,
+	*serving.ExternalFunctionRequest,
+)
+
+func newHttpRequest() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var httpRequestReq serving.ExternalFunctionRequest
+
+	// TODO: short flags
+
+	cmd.Flags().StringVar(&httpRequestReq.Headers, "headers", httpRequestReq.Headers, `Additional headers for the request.`)
+	cmd.Flags().StringVar(&httpRequestReq.Json, "json", httpRequestReq.Json, `The JSON payload to send in the request body.`)
+	cmd.Flags().StringVar(&httpRequestReq.Params, "params", httpRequestReq.Params, `Query parameters for the request.`)
+
+	cmd.Use = "http-request CONNECTION_NAME METHOD PATH"
+	cmd.Short = `Make external services call using the credentials stored in UC Connection.`
+	cmd.Long = `Make external services call using the credentials stored in UC Connection.
+
+  Arguments:
+    CONNECTION_NAME: The connection name to use. This is required to identify the external
+      connection.
+    METHOD: The HTTP method to use (e.g., 'GET', 'POST').
+    PATH: The relative path for the API endpoint. This is required.`
+
+	// This command is being previewed; hide from help output.
+	cmd.Hidden = true
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(3)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustWorkspaceClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		w := root.WorkspaceClient(ctx)
+
+		httpRequestReq.ConnectionName = args[0]
+		_, err = fmt.Sscan(args[1], &httpRequestReq.Method)
+		if err != nil {
+			return fmt.Errorf("invalid METHOD: %s", args[1])
+		}
+		httpRequestReq.Path = args[2]
+
+		response, err := w.ServingEndpoints.HttpRequest(ctx, httpRequestReq)
+		if err != nil {
+			return err
+		}
+		return cmdio.Render(ctx, response)
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range httpRequestOverrides {
+		fn(cmd, &httpRequestReq)
+	}
+
+	return cmd
+}
+
 // start list command
 
 // Slice with functions to override default command behavior.
@@ -849,7 +938,7 @@ func newPutAiGateway() *cobra.Command {
 	cmd.Long = `Update AI Gateway of a serving endpoint.
   
   Used to update the AI Gateway of a serving endpoint. NOTE: Only external model
-  endpoints are currently supported.
+  and provisioned throughput endpoints are currently supported.
 
   Arguments:
     NAME: The name of the serving endpoint whose AI Gateway is being updated. This
diff --git a/go.mod b/go.mod
index 930963f89..bd8997190 100644
--- a/go.mod
+++ b/go.mod
@@ -8,7 +8,7 @@ require (
 	github.com/BurntSushi/toml v1.4.0 // MIT
 	github.com/Masterminds/semver/v3 v3.3.1 // MIT
 	github.com/briandowns/spinner v1.23.1 // Apache 2.0
-	github.com/databricks/databricks-sdk-go v0.55.0 // Apache 2.0
+	github.com/databricks/databricks-sdk-go v0.56.1 // Apache 2.0
 	github.com/fatih/color v1.18.0 // MIT
 	github.com/google/uuid v1.6.0 // BSD-3-Clause
 	github.com/hashicorp/go-version v1.7.0 // MPL 2.0
diff --git a/go.sum b/go.sum
index d025b3947..dec1d40b2 100644
--- a/go.sum
+++ b/go.sum
@@ -34,8 +34,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cyphar/filepath-securejoin v0.2.5 h1:6iR5tXJ/e6tJZzzdMc1km3Sa7RRIVBKAK32O2s7AYfo=
 github.com/cyphar/filepath-securejoin v0.2.5/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
-github.com/databricks/databricks-sdk-go v0.55.0 h1:ReziD6spzTDltM0ml80LggKo27F3oUjgTinCFDJDnak=
-github.com/databricks/databricks-sdk-go v0.55.0/go.mod h1:JpLizplEs+up9/Z4Xf2x++o3sM9eTTWFGzIXAptKJzI=
+github.com/databricks/databricks-sdk-go v0.56.1 h1:sgweTRvAQaI8EPrfDnVdAB0lNX6L5uTT720SlMMQI2U=
+github.com/databricks/databricks-sdk-go v0.56.1/go.mod h1:JpLizplEs+up9/Z4Xf2x++o3sM9eTTWFGzIXAptKJzI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
diff --git a/integration/cmd/sync/sync_test.go b/integration/cmd/sync/sync_test.go
index 632497054..88e6ed89a 100644
--- a/integration/cmd/sync/sync_test.go
+++ b/integration/cmd/sync/sync_test.go
@@ -158,7 +158,7 @@ func (a *syncTest) remoteFileContent(ctx context.Context, relativePath, expected
 
 	var res []byte
 	a.c.Eventually(func() bool {
-		err = apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, &res)
+		err = apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, nil, &res)
 		require.NoError(a.t, err)
 		actualContent := string(res)
 		return actualContent == expectedContent
diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go
index 88bbadd32..7102b6e29 100644
--- a/libs/filer/files_client.go
+++ b/libs/filer/files_client.go
@@ -148,7 +148,7 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader,
 	overwrite := slices.Contains(mode, OverwriteIfExists)
 	urlPath = fmt.Sprintf("%s?overwrite=%t", urlPath, overwrite)
 	headers := map[string]string{"Content-Type": "application/octet-stream"}
-	err = w.apiClient.Do(ctx, http.MethodPut, urlPath, headers, reader, nil)
+	err = w.apiClient.Do(ctx, http.MethodPut, urlPath, headers, nil, reader, nil)
 
 	// Return early on success.
 	if err == nil {
@@ -176,7 +176,7 @@ func (w *FilesClient) Read(ctx context.Context, name string) (io.ReadCloser, err
 	}
 
 	var reader io.ReadCloser
-	err = w.apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, &reader)
+	err = w.apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, nil, &reader)
 
 	// Return early on success.
 	if err == nil {
diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go
index 8d5148edd..1d514f13b 100644
--- a/libs/filer/workspace_files_client.go
+++ b/libs/filer/workspace_files_client.go
@@ -106,7 +106,7 @@ func (info *wsfsFileInfo) MarshalJSON() ([]byte, error) {
 // as an interface to allow for mocking in tests.
 type apiClient interface {
 	Do(ctx context.Context, method, path string,
-		headers map[string]string, request, response any,
+		headers map[string]string, queryString map[string]any, request, response any,
 		visitors ...func(*http.Request) error) error
 }
 
@@ -156,7 +156,7 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io
 		return err
 	}
 
-	err = w.apiClient.Do(ctx, http.MethodPost, urlPath, nil, body, nil)
+	err = w.apiClient.Do(ctx, http.MethodPost, urlPath, nil, nil, body, nil)
 
 	// Return early on success.
 	if err == nil {
@@ -341,6 +341,7 @@ func (w *WorkspaceFilesClient) Stat(ctx context.Context, name string) (fs.FileIn
 		http.MethodGet,
 		"/api/2.0/workspace/get-status",
 		nil,
+		nil,
 		map[string]string{
 			"path":               absPath,
 			"return_export_info": "true",
diff --git a/libs/filer/workspace_files_extensions_client_test.go b/libs/filer/workspace_files_extensions_client_test.go
index 9ea837fa9..e9fde4762 100644
--- a/libs/filer/workspace_files_extensions_client_test.go
+++ b/libs/filer/workspace_files_extensions_client_test.go
@@ -17,7 +17,7 @@ type mockApiClient struct {
 }
 
 func (m *mockApiClient) Do(ctx context.Context, method, path string,
-	headers map[string]string, request, response any,
+	headers map[string]string, queryString map[string]any, request, response any,
 	visitors ...func(*http.Request) error,
 ) error {
 	args := m.Called(ctx, method, path, headers, request, response, visitors)
diff --git a/libs/git/info.go b/libs/git/info.go
index 46e57be48..dc4af9b6d 100644
--- a/libs/git/info.go
+++ b/libs/git/info.go
@@ -66,6 +66,7 @@ func fetchRepositoryInfoAPI(ctx context.Context, path string, w *databricks.Work
 		http.MethodGet,
 		apiEndpoint,
 		nil,
+		nil,
 		map[string]string{
 			"path":            path,
 			"return_git_info": "true",

From 65fbbd9a7c75a2404fa3d4956560ab037535d779 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 27 Jan 2025 14:22:08 +0100
Subject: [PATCH 152/247] libs/python: Remove DetectInterpreters (#2234)

## Changes
- Remove DetectInterpreters from DetectExecutable call: python3 or
python should always be on on the PATH. We don't need to detect
non-standard situations like python3.10 is present but python3 is not.
- I moved DetectInterpreters to cmd/labs where it is still used.

This is a follow up to https://github.com/databricks/cli/pull/2034

## Tests
Existing tests.
---
 cmd/labs/project/installer.go                 |  3 +--
 .../labs/project}/interpreters.go             |  2 +-
 .../labs/project}/interpreters_unix_test.go   |  2 +-
 .../labs/project}/interpreters_win_test.go    |  2 +-
 .../testdata/other-binaries-filtered/python   |  0
 .../other-binaries-filtered/python3-whatever  |  0
 .../other-binaries-filtered/python3.10        |  0
 .../other-binaries-filtered/python3.10.100    |  0
 .../other-binaries-filtered/python3.11        |  0
 .../other-binaries-filtered/python4.8         |  0
 .../testdata/other-binaries-filtered/python5  |  0
 .../testdata/other-binaries-filtered/python6  |  0
 .../testdata/other-binaries-filtered/python7  |  0
 .../testdata/other-binaries-filtered/pythonw  |  0
 .../other-binaries-filtered/real-python3.11.4 |  0
 .../testdata/other-binaries-filtered/whatever |  0
 .../testdata/world-writeable/python8.4        |  0
 libs/python/detect.go                         | 22 +------------------
 libs/python/detect_unix_test.go               | 12 ++--------
 libs/python/detect_win_test.go                |  2 +-
 20 files changed, 8 insertions(+), 37 deletions(-)
 rename {libs/python => cmd/labs/project}/interpreters.go (99%)
 rename {libs/python => cmd/labs/project}/interpreters_unix_test.go (99%)
 rename {libs/python => cmd/labs/project}/interpreters_win_test.go (97%)
 rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/python (100%)
 rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/python3-whatever (100%)
 rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/python3.10 (100%)
 rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/python3.10.100 (100%)
 rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/python3.11 (100%)
 rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/python4.8 (100%)
 rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/python5 (100%)
 rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/python6 (100%)
 rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/python7 (100%)
 rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/pythonw (100%)
 rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/real-python3.11.4 (100%)
 rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/whatever (100%)
 rename {libs/python => cmd/labs/project}/testdata/world-writeable/python8.4 (100%)

diff --git a/cmd/labs/project/installer.go b/cmd/labs/project/installer.go
index 7d31623bb..05f7d68aa 100644
--- a/cmd/labs/project/installer.go
+++ b/cmd/labs/project/installer.go
@@ -15,7 +15,6 @@ import (
 	"github.com/databricks/cli/libs/databrickscfg/profile"
 	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/cli/libs/process"
-	"github.com/databricks/cli/libs/python"
 	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/sql"
@@ -223,7 +222,7 @@ func (i *installer) setupPythonVirtualEnvironment(ctx context.Context, w *databr
 	feedback := cmdio.Spinner(ctx)
 	defer close(feedback)
 	feedback <- "Detecting all installed Python interpreters on the system"
-	pythonInterpreters, err := python.DetectInterpreters(ctx)
+	pythonInterpreters, err := DetectInterpreters(ctx)
 	if err != nil {
 		return fmt.Errorf("detect: %w", err)
 	}
diff --git a/libs/python/interpreters.go b/cmd/labs/project/interpreters.go
similarity index 99%
rename from libs/python/interpreters.go
rename to cmd/labs/project/interpreters.go
index 6071309a8..00f099ed4 100644
--- a/libs/python/interpreters.go
+++ b/cmd/labs/project/interpreters.go
@@ -1,4 +1,4 @@
-package python
+package project
 
 import (
 	"context"
diff --git a/libs/python/interpreters_unix_test.go b/cmd/labs/project/interpreters_unix_test.go
similarity index 99%
rename from libs/python/interpreters_unix_test.go
rename to cmd/labs/project/interpreters_unix_test.go
index 57adc9279..a5bbb6468 100644
--- a/libs/python/interpreters_unix_test.go
+++ b/cmd/labs/project/interpreters_unix_test.go
@@ -1,6 +1,6 @@
 //go:build unix
 
-package python
+package project
 
 import (
 	"context"
diff --git a/libs/python/interpreters_win_test.go b/cmd/labs/project/interpreters_win_test.go
similarity index 97%
rename from libs/python/interpreters_win_test.go
rename to cmd/labs/project/interpreters_win_test.go
index f99981529..2316daa30 100644
--- a/libs/python/interpreters_win_test.go
+++ b/cmd/labs/project/interpreters_win_test.go
@@ -1,6 +1,6 @@
 //go:build windows
 
-package python
+package project
 
 import (
 	"context"
diff --git a/libs/python/testdata/other-binaries-filtered/python b/cmd/labs/project/testdata/other-binaries-filtered/python
similarity index 100%
rename from libs/python/testdata/other-binaries-filtered/python
rename to cmd/labs/project/testdata/other-binaries-filtered/python
diff --git a/libs/python/testdata/other-binaries-filtered/python3-whatever b/cmd/labs/project/testdata/other-binaries-filtered/python3-whatever
similarity index 100%
rename from libs/python/testdata/other-binaries-filtered/python3-whatever
rename to cmd/labs/project/testdata/other-binaries-filtered/python3-whatever
diff --git a/libs/python/testdata/other-binaries-filtered/python3.10 b/cmd/labs/project/testdata/other-binaries-filtered/python3.10
similarity index 100%
rename from libs/python/testdata/other-binaries-filtered/python3.10
rename to cmd/labs/project/testdata/other-binaries-filtered/python3.10
diff --git a/libs/python/testdata/other-binaries-filtered/python3.10.100 b/cmd/labs/project/testdata/other-binaries-filtered/python3.10.100
similarity index 100%
rename from libs/python/testdata/other-binaries-filtered/python3.10.100
rename to cmd/labs/project/testdata/other-binaries-filtered/python3.10.100
diff --git a/libs/python/testdata/other-binaries-filtered/python3.11 b/cmd/labs/project/testdata/other-binaries-filtered/python3.11
similarity index 100%
rename from libs/python/testdata/other-binaries-filtered/python3.11
rename to cmd/labs/project/testdata/other-binaries-filtered/python3.11
diff --git a/libs/python/testdata/other-binaries-filtered/python4.8 b/cmd/labs/project/testdata/other-binaries-filtered/python4.8
similarity index 100%
rename from libs/python/testdata/other-binaries-filtered/python4.8
rename to cmd/labs/project/testdata/other-binaries-filtered/python4.8
diff --git a/libs/python/testdata/other-binaries-filtered/python5 b/cmd/labs/project/testdata/other-binaries-filtered/python5
similarity index 100%
rename from libs/python/testdata/other-binaries-filtered/python5
rename to cmd/labs/project/testdata/other-binaries-filtered/python5
diff --git a/libs/python/testdata/other-binaries-filtered/python6 b/cmd/labs/project/testdata/other-binaries-filtered/python6
similarity index 100%
rename from libs/python/testdata/other-binaries-filtered/python6
rename to cmd/labs/project/testdata/other-binaries-filtered/python6
diff --git a/libs/python/testdata/other-binaries-filtered/python7 b/cmd/labs/project/testdata/other-binaries-filtered/python7
similarity index 100%
rename from libs/python/testdata/other-binaries-filtered/python7
rename to cmd/labs/project/testdata/other-binaries-filtered/python7
diff --git a/libs/python/testdata/other-binaries-filtered/pythonw b/cmd/labs/project/testdata/other-binaries-filtered/pythonw
similarity index 100%
rename from libs/python/testdata/other-binaries-filtered/pythonw
rename to cmd/labs/project/testdata/other-binaries-filtered/pythonw
diff --git a/libs/python/testdata/other-binaries-filtered/real-python3.11.4 b/cmd/labs/project/testdata/other-binaries-filtered/real-python3.11.4
similarity index 100%
rename from libs/python/testdata/other-binaries-filtered/real-python3.11.4
rename to cmd/labs/project/testdata/other-binaries-filtered/real-python3.11.4
diff --git a/libs/python/testdata/other-binaries-filtered/whatever b/cmd/labs/project/testdata/other-binaries-filtered/whatever
similarity index 100%
rename from libs/python/testdata/other-binaries-filtered/whatever
rename to cmd/labs/project/testdata/other-binaries-filtered/whatever
diff --git a/libs/python/testdata/world-writeable/python8.4 b/cmd/labs/project/testdata/world-writeable/python8.4
similarity index 100%
rename from libs/python/testdata/world-writeable/python8.4
rename to cmd/labs/project/testdata/world-writeable/python8.4
diff --git a/libs/python/detect.go b/libs/python/detect.go
index e86d9d621..75158da65 100644
--- a/libs/python/detect.go
+++ b/libs/python/detect.go
@@ -39,27 +39,7 @@ func DetectExecutable(ctx context.Context) (string, error) {
 	//
 	// See https://github.com/pyenv/pyenv#understanding-python-version-selection
 
-	out, err := exec.LookPath(GetExecutable())
-
-	// most of the OS'es have python3 in $PATH, but for those which don't,
-	// we perform the latest version lookup
-	if err != nil && !errors.Is(err, exec.ErrNotFound) {
-		return "", err
-	}
-	if out != "" {
-		return out, nil
-	}
-	// otherwise, detect all interpreters and pick the least that satisfies
-	// minimal version requirements
-	all, err := DetectInterpreters(ctx)
-	if err != nil {
-		return "", err
-	}
-	interpreter, err := all.AtLeast("3.8")
-	if err != nil {
-		return "", err
-	}
-	return interpreter.Path, nil
+	return exec.LookPath(GetExecutable())
 }
 
 // DetectVEnvExecutable returns the path to the python3 executable inside venvPath,
diff --git a/libs/python/detect_unix_test.go b/libs/python/detect_unix_test.go
index a962e1f55..1774aa108 100644
--- a/libs/python/detect_unix_test.go
+++ b/libs/python/detect_unix_test.go
@@ -16,24 +16,16 @@ func TestDetectsViaPathLookup(t *testing.T) {
 	assert.NotEmpty(t, py)
 }
 
-func TestDetectsViaListing(t *testing.T) {
-	t.Setenv("PATH", "testdata/other-binaries-filtered")
-	ctx := context.Background()
-	py, err := DetectExecutable(ctx)
-	assert.NoError(t, err)
-	assert.Equal(t, "testdata/other-binaries-filtered/python3.10", py)
-}
-
 func TestDetectFailsNoInterpreters(t *testing.T) {
 	t.Setenv("PATH", "testdata")
 	ctx := context.Background()
 	_, err := DetectExecutable(ctx)
-	assert.Equal(t, ErrNoPythonInterpreters, err)
+	assert.Error(t, err)
 }
 
 func TestDetectFailsNoMinimalVersion(t *testing.T) {
 	t.Setenv("PATH", "testdata/no-python3")
 	ctx := context.Background()
 	_, err := DetectExecutable(ctx)
-	assert.EqualError(t, err, "cannot find Python greater or equal to v3.8.0")
+	assert.Error(t, err)
 }
diff --git a/libs/python/detect_win_test.go b/libs/python/detect_win_test.go
index 2ef811a4b..7b2ee281e 100644
--- a/libs/python/detect_win_test.go
+++ b/libs/python/detect_win_test.go
@@ -20,5 +20,5 @@ func TestDetectFailsNoInterpreters(t *testing.T) {
 	t.Setenv("PATH", "testdata")
 	ctx := context.Background()
 	_, err := DetectExecutable(ctx)
-	assert.ErrorIs(t, err, ErrNoPythonInterpreters)
+	assert.Error(t, err)
 }

From 52bf7e388a80beb95d248dc623cfda3cf5d5e137 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 27 Jan 2025 15:25:56 +0100
Subject: [PATCH 153/247] acc: Propagate user's UV_CACHE_DIR to tests (#2239)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

There is a speed up in 0.5s but it is still 4.4s, so something else is
slow there.

Benchmarking bundle/templates/experimental-jobs-as-code:

```
# Without UV_CACHE_DIR
~/work/cli/acceptance/bundle/templates/experimental-jobs-as-code % hyperfine --warmup 2 'testme -count=1'
Benchmark 1: testme -count=1
  Time (mean ± σ):      4.950 s ±  0.079 s    [User: 2.730 s, System: 8.524 s]
  Range (min … max):    4.838 s …  5.076 s    10 runs

# With UV_CACHE_DIR
~/work/cli/acceptance/bundle/templates/experimental-jobs-as-code % hyperfine --warmup 2 'testme -count=1'
Benchmark 1: testme -count=1
  Time (mean ± σ):      4.410 s ±  0.049 s    [User: 2.669 s, System: 8.710 s]
  Range (min … max):    4.324 s …  4.467 s    10 runs
```
---
 acceptance/acceptance_test.go                   | 17 +++++++++++++++++
 .../templates/experimental-jobs-as-code/script  |  2 +-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index e48bd9908..47295b47a 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -100,6 +100,10 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	// Prevent CLI from downloading terraform in each test:
 	t.Setenv("DATABRICKS_TF_EXEC_PATH", tempHomeDir)
 
+	// Make use of uv cache; since we set HomeEnvVar to temporary directory, it is not picked up automatically
+	uvCache := getUVDefaultCacheDir(t)
+	t.Setenv("UV_CACHE_DIR", uvCache)
+
 	ctx := context.Background()
 	cloudEnv := os.Getenv("CLOUD_ENV")
 
@@ -486,3 +490,16 @@ func ListDir(t *testing.T, src string) []string {
 	}
 	return files
 }
+
+func getUVDefaultCacheDir(t *testing.T) string {
+	// According to uv docs https://docs.astral.sh/uv/concepts/cache/#caching-in-continuous-integration
+	// the default cache directory is
+	// "A system-appropriate cache directory, e.g., $XDG_CACHE_HOME/uv or $HOME/.cache/uv on Unix and %LOCALAPPDATA%\uv\cache on Windows"
+	cacheDir, err := os.UserCacheDir()
+	require.NoError(t, err)
+	if runtime.GOOS == "windows" {
+		return cacheDir + "\\uv\\cache"
+	} else {
+		return cacheDir + "/uv"
+	}
+}
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/script b/acceptance/bundle/templates/experimental-jobs-as-code/script
index af28b9d0a..0223b3326 100644
--- a/acceptance/bundle/templates/experimental-jobs-as-code/script
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/script
@@ -3,7 +3,7 @@ trace $CLI bundle init experimental-jobs-as-code --config-file ./input.json --ou
 cd output/my_jobs_as_code
 
 # silence uv output because it's non-deterministic
-uv sync 2> /dev/null
+uv sync -q
 
 # remove version constraint because it always creates a warning on dev builds
 cat databricks.yml | grep -v databricks_cli_version > databricks.yml.new

From 67d1413db5b84df6643f3c1571abae13da14c6e2 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 27 Jan 2025 15:34:53 +0100
Subject: [PATCH 154/247] Add default regex for DEV_VERSION (#2241)

## Changes

- Replace development version with $DEV_VERSION
- Update experimental-jobs-as-code to make use of it.

## Tests
- Existing tests.
- Using this in https://github.com/databricks/cli/pull/2213
---
 acceptance/acceptance_test.go                              | 1 +
 .../bundle/templates/experimental-jobs-as-code/output.txt  | 2 ++
 .../output/my_jobs_as_code/databricks.yml                  | 1 +
 .../bundle/templates/experimental-jobs-as-code/script      | 4 ----
 acceptance/selftest/output.txt                             | 4 ++++
 acceptance/selftest/script                                 | 3 +++
 libs/testdiff/replacement.go                               | 7 +++++++
 7 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 47295b47a..5eb08f674 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -128,6 +128,7 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	testdiff.PrepareReplacementsUser(t, &repls, *user)
 	testdiff.PrepareReplacementsWorkspaceClient(t, &repls, workspaceClient)
 	testdiff.PrepareReplacementsUUID(t, &repls)
+	testdiff.PrepareReplacementsDevVersion(t, &repls)
 
 	testDirs := getTests(t)
 	require.NotEmpty(t, testDirs)
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output.txt b/acceptance/bundle/templates/experimental-jobs-as-code/output.txt
index 1aa8a94d5..10aca003e 100644
--- a/acceptance/bundle/templates/experimental-jobs-as-code/output.txt
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output.txt
@@ -10,6 +10,8 @@ Please refer to the README.md file for "getting started" instructions.
 See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.
 
 >>> $CLI bundle validate -t dev --output json
+Warning: Ignoring Databricks CLI version constraint for development build. Required: >= 0.238.0, current: $DEV_VERSION
+
 {
   "jobs": {
     "my_jobs_as_code_job": {
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml
index fd87aa381..a1a93d95c 100644
--- a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml
@@ -3,6 +3,7 @@
 bundle:
   name: my_jobs_as_code
   uuid: <UUID>
+  databricks_cli_version: ">= 0.238.0"
 
 experimental:
   python:
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/script b/acceptance/bundle/templates/experimental-jobs-as-code/script
index 0223b3326..10188aabd 100644
--- a/acceptance/bundle/templates/experimental-jobs-as-code/script
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/script
@@ -5,10 +5,6 @@ cd output/my_jobs_as_code
 # silence uv output because it's non-deterministic
 uv sync -q
 
-# remove version constraint because it always creates a warning on dev builds
-cat databricks.yml | grep -v databricks_cli_version > databricks.yml.new
-mv databricks.yml.new databricks.yml
-
 trace $CLI bundle validate -t dev --output json | jq ".resources"
 
 rm -fr .venv resources/__pycache__ uv.lock my_jobs_as_code.egg-info
diff --git a/acceptance/selftest/output.txt b/acceptance/selftest/output.txt
index 9fdfbc1e7..91aa8c33e 100644
--- a/acceptance/selftest/output.txt
+++ b/acceptance/selftest/output.txt
@@ -33,3 +33,7 @@ $TMPDIR/subdir/a/b/c
 1234
 CUSTOM_NUMBER_REGEX
 123456
+
+=== Testing --version
+>>> $CLI --version
+Databricks CLI v$DEV_VERSION
diff --git a/acceptance/selftest/script b/acceptance/selftest/script
index 665726167..bccf30e71 100644
--- a/acceptance/selftest/script
+++ b/acceptance/selftest/script
@@ -24,3 +24,6 @@ printf "\n=== Custom regex can be specified in [[Repl]] section\n"
 echo 1234
 echo 12345
 echo 123456
+
+printf "\n=== Testing --version"
+trace $CLI --version
diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go
index b512374a3..40e7e72b4 100644
--- a/libs/testdiff/replacement.go
+++ b/libs/testdiff/replacement.go
@@ -23,6 +23,8 @@ var (
 	uuidRegex        = regexp.MustCompile(`[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}`)
 	numIdRegex       = regexp.MustCompile(`[0-9]{3,}`)
 	privatePathRegex = regexp.MustCompile(`(/tmp|/private)(/.*)/([a-zA-Z0-9]+)`)
+	// Version could v0.0.0-dev+21e1aacf518a or just v0.0.0-dev (the latter is currently the case on Windows)
+	devVersionRegex = regexp.MustCompile(`0\.0\.0-dev(\+[a-f0-9]{10,16})?`)
 )
 
 type Replacement struct {
@@ -211,3 +213,8 @@ func PrepareReplacementsTemporaryDirectory(t testutil.TestingT, r *ReplacementsC
 	t.Helper()
 	r.append(privatePathRegex, "/tmp/.../$3")
 }
+
+func PrepareReplacementsDevVersion(t testutil.TestingT, r *ReplacementsContext) {
+	t.Helper()
+	r.append(devVersionRegex, "$$DEV_VERSION")
+}

From be908ee1a17abe36c573a24ac83033243c154379 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 27 Jan 2025 16:28:33 +0100
Subject: [PATCH 155/247] Add acceptance test for 'experimental.scripts'
 (#2240)

---
 acceptance/bundle/scripts/databricks.yml | 11 +++++
 acceptance/bundle/scripts/myscript.py    |  8 ++++
 acceptance/bundle/scripts/output.txt     | 52 ++++++++++++++++++++++++
 acceptance/bundle/scripts/script         |  3 ++
 acceptance/server_test.go                |  4 ++
 bundle/scripts/scripts_test.go           | 51 -----------------------
 6 files changed, 78 insertions(+), 51 deletions(-)
 create mode 100644 acceptance/bundle/scripts/databricks.yml
 create mode 100644 acceptance/bundle/scripts/myscript.py
 create mode 100644 acceptance/bundle/scripts/output.txt
 create mode 100644 acceptance/bundle/scripts/script
 delete mode 100644 bundle/scripts/scripts_test.go

diff --git a/acceptance/bundle/scripts/databricks.yml b/acceptance/bundle/scripts/databricks.yml
new file mode 100644
index 000000000..6421e2b59
--- /dev/null
+++ b/acceptance/bundle/scripts/databricks.yml
@@ -0,0 +1,11 @@
+bundle:
+  name: scripts
+
+experimental:
+  scripts:
+    preinit: "python3 ./myscript.py $EXITCODE preinit"
+    postinit: "python3 ./myscript.py 0 postinit"
+    prebuild: "python3 ./myscript.py 0 prebuild"
+    postbuild: "python3 ./myscript.py 0 postbuild"
+    predeploy: "python3 ./myscript.py 0 predeploy"
+    postdeploy: "python3 ./myscript.py 0 postdeploy"
diff --git a/acceptance/bundle/scripts/myscript.py b/acceptance/bundle/scripts/myscript.py
new file mode 100644
index 000000000..d10f497e1
--- /dev/null
+++ b/acceptance/bundle/scripts/myscript.py
@@ -0,0 +1,8 @@
+import sys
+
+info = " ".join(sys.argv[1:])
+sys.stderr.write(f"from myscript.py {info}: hello stderr!\n")
+sys.stdout.write(f"from myscript.py {info}: hello stdout!\n")
+
+exitcode = int(sys.argv[1])
+sys.exit(exitcode)
diff --git a/acceptance/bundle/scripts/output.txt b/acceptance/bundle/scripts/output.txt
new file mode 100644
index 000000000..ec5978380
--- /dev/null
+++ b/acceptance/bundle/scripts/output.txt
@@ -0,0 +1,52 @@
+
+>>> EXITCODE=0 errcode $CLI bundle validate
+Executing 'preinit' script
+from myscript.py 0 preinit: hello stdout!
+from myscript.py 0 preinit: hello stderr!
+Executing 'postinit' script
+from myscript.py 0 postinit: hello stdout!
+from myscript.py 0 postinit: hello stderr!
+Name: scripts
+Target: default
+Workspace:
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/scripts/default
+
+Validation OK!
+
+>>> EXITCODE=1 errcode $CLI bundle validate
+Executing 'preinit' script
+from myscript.py 1 preinit: hello stdout!
+from myscript.py 1 preinit: hello stderr!
+Error: failed to execute script: exit status 1
+
+Name: scripts
+
+Found 1 error
+
+Exit code: 1
+
+>>> EXITCODE=0 errcode $CLI bundle deploy
+Executing 'preinit' script
+from myscript.py 0 preinit: hello stdout!
+from myscript.py 0 preinit: hello stderr!
+Executing 'postinit' script
+from myscript.py 0 postinit: hello stdout!
+from myscript.py 0 postinit: hello stderr!
+Executing 'prebuild' script
+from myscript.py 0 prebuild: hello stdout!
+from myscript.py 0 prebuild: hello stderr!
+Executing 'postbuild' script
+from myscript.py 0 postbuild: hello stdout!
+from myscript.py 0 postbuild: hello stderr!
+Executing 'predeploy' script
+from myscript.py 0 predeploy: hello stdout!
+from myscript.py 0 predeploy: hello stderr!
+Error: unable to deploy to /Workspace/Users/$USERNAME/.bundle/scripts/default/state as $USERNAME.
+Please make sure the current user or one of their groups is listed under the permissions of this bundle.
+For assistance, contact the owners of this project.
+They may need to redeploy the bundle to apply the new permissions.
+Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.
+
+
+Exit code: 1
diff --git a/acceptance/bundle/scripts/script b/acceptance/bundle/scripts/script
new file mode 100644
index 000000000..de07d277e
--- /dev/null
+++ b/acceptance/bundle/scripts/script
@@ -0,0 +1,3 @@
+trace EXITCODE=0 errcode $CLI bundle validate
+trace EXITCODE=1 errcode $CLI bundle validate
+trace EXITCODE=0 errcode $CLI bundle deploy
diff --git a/acceptance/server_test.go b/acceptance/server_test.go
index eb8cbb24a..dbc55c03f 100644
--- a/acceptance/server_test.go
+++ b/acceptance/server_test.go
@@ -146,4 +146,8 @@ func AddHandlers(server *TestServer) {
 			},
 		}, nil
 	})
+
+	server.Handle("POST /api/2.0/workspace/mkdirs", func(r *http.Request) (any, error) {
+		return "{}", nil
+	})
 }
diff --git a/bundle/scripts/scripts_test.go b/bundle/scripts/scripts_test.go
deleted file mode 100644
index 0c92bc2c3..000000000
--- a/bundle/scripts/scripts_test.go
+++ /dev/null
@@ -1,51 +0,0 @@
-package scripts
-
-import (
-	"bufio"
-	"context"
-	"strings"
-	"testing"
-
-	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/bundle/config"
-	"github.com/databricks/cli/libs/exec"
-	"github.com/stretchr/testify/require"
-)
-
-func TestExecutesHook(t *testing.T) {
-	b := &bundle.Bundle{
-		Config: config.Root{
-			Experimental: &config.Experimental{
-				Scripts: map[config.ScriptHook]config.Command{
-					config.ScriptPreBuild: "echo 'Hello'",
-				},
-			},
-		},
-	}
-
-	executor, err := exec.NewCommandExecutor(b.BundleRootPath)
-	require.NoError(t, err)
-	_, out, err := executeHook(context.Background(), executor, b, config.ScriptPreBuild)
-	require.NoError(t, err)
-
-	reader := bufio.NewReader(out)
-	line, err := reader.ReadString('\n')
-
-	require.NoError(t, err)
-	require.Equal(t, "Hello", strings.TrimSpace(line))
-}
-
-func TestExecuteMutator(t *testing.T) {
-	b := &bundle.Bundle{
-		Config: config.Root{
-			Experimental: &config.Experimental{
-				Scripts: map[config.ScriptHook]config.Command{
-					config.ScriptPreBuild: "echo 'Hello'",
-				},
-			},
-		},
-	}
-
-	diags := bundle.Apply(context.Background(), b, Execute(config.ScriptPreInit))
-	require.NoError(t, diags.Error())
-}

From 60709e3d48a711b931d341196120f4450ee78499 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 28 Jan 2025 11:15:32 +0100
Subject: [PATCH 156/247] acc: Restore unexpected output error (#2243)

## Changes
Restore original behaviour of acceptance tests: any unaccounted for
files trigger an error (not just those that start with "out"). This got
changed in
https://github.com/databricks/cli/pull/2146/files#diff-2bb968d823f4afb825e1dcea2879bdbdedf2b7c15d4e77f47905691b14246a04L196
which started only checking files starting with "out*" and skipping
everything else.

## Tests
Existing tests.
---
 acceptance/acceptance_test.go                   | 1 +
 acceptance/bundle/git-permerror/script          | 3 ++-
 acceptance/bundle/syncroot/dotdot-git/script    | 4 +++-
 acceptance/bundle/syncroot/dotdot-git/test.toml | 3 +++
 4 files changed, 9 insertions(+), 2 deletions(-)
 create mode 100644 acceptance/bundle/syncroot/dotdot-git/test.toml

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 5eb08f674..2d67fb269 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -245,6 +245,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		if _, ok := outputs[relPath]; ok {
 			continue
 		}
+		t.Errorf("Unexpected output: %s", relPath)
 		if strings.HasPrefix(relPath, "out") {
 			// We have a new file starting with "out"
 			// Show the contents & support overwrite mode for it:
diff --git a/acceptance/bundle/git-permerror/script b/acceptance/bundle/git-permerror/script
index 782cbf5bc..3a9b4db24 100644
--- a/acceptance/bundle/git-permerror/script
+++ b/acceptance/bundle/git-permerror/script
@@ -22,4 +22,5 @@ trace chmod 000 .git/config
 errcode trace $CLI bundle validate -o json | jq .bundle.git
 errcode trace withdir subdir/a/b $CLI bundle validate -o json | jq .bundle.git
 
-rm -fr .git
+cd ..
+rm -fr myrepo
diff --git a/acceptance/bundle/syncroot/dotdot-git/script b/acceptance/bundle/syncroot/dotdot-git/script
index 0706a1d5e..278e77101 100644
--- a/acceptance/bundle/syncroot/dotdot-git/script
+++ b/acceptance/bundle/syncroot/dotdot-git/script
@@ -3,4 +3,6 @@ mkdir myrepo
 cd myrepo
 cp ../databricks.yml .
 git-repo-init
-$CLI bundle validate | sed 's/\\\\/\//g'
+errcode $CLI bundle validate
+cd ..
+rm -fr myrepo
diff --git a/acceptance/bundle/syncroot/dotdot-git/test.toml b/acceptance/bundle/syncroot/dotdot-git/test.toml
new file mode 100644
index 000000000..f57f83ee4
--- /dev/null
+++ b/acceptance/bundle/syncroot/dotdot-git/test.toml
@@ -0,0 +1,3 @@
+[[Repls]]
+Old = '\\\\myrepo'
+New = '/myrepo'

From 11436faafe5361bd390fa04dc699807e31db6144 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 28 Jan 2025 11:22:29 +0100
Subject: [PATCH 157/247] acc: Avoid reading and applying replacements on large
 files; validate utf8 (#2244)

## Changes
- Do not start replacement / comparison if file is too large or not
valid utf-8.
- This helps to prevent replacements if there is accidentally a large
binary (e.g. terraform).

## Tests
Found this problem when working on
https://github.com/databricks/cli/pull/2242 -- the tests tried to
applied replacements on terraform binary and crashed. With this change,
an error is reported instead.
---
 acceptance/acceptance_test.go | 51 ++++++++++++++++++++++++-----------
 1 file changed, 35 insertions(+), 16 deletions(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 2d67fb269..877c7239d 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -15,6 +15,7 @@ import (
 	"strings"
 	"testing"
 	"time"
+	"unicode/utf8"
 
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/env"
@@ -44,6 +45,7 @@ const (
 	EntryPointScript = "script"
 	CleanupScript    = "script.cleanup"
 	PrepareScript    = "script.prepare"
+	MaxFileSize      = 100_000
 )
 
 var Scripts = map[string]bool{
@@ -257,15 +259,15 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool) {
 	pathRef := filepath.Join(dirRef, relPath)
 	pathNew := filepath.Join(dirNew, relPath)
-	bufRef, okRef := readIfExists(t, pathRef)
-	bufNew, okNew := readIfExists(t, pathNew)
+	bufRef, okRef := tryReading(t, pathRef)
+	bufNew, okNew := tryReading(t, pathNew)
 	if !okRef && !okNew {
-		t.Errorf("Both files are missing: %s, %s", pathRef, pathNew)
+		t.Errorf("Both files are missing or have errors: %s, %s", pathRef, pathNew)
 		return
 	}
 
-	valueRef := testdiff.NormalizeNewlines(string(bufRef))
-	valueNew := testdiff.NormalizeNewlines(string(bufNew))
+	valueRef := testdiff.NormalizeNewlines(bufRef)
+	valueNew := testdiff.NormalizeNewlines(bufNew)
 
 	// Apply replacements to the new value only.
 	// The reference value is stored after applying replacements.
@@ -323,14 +325,14 @@ func readMergedScriptContents(t *testing.T, dir string) string {
 	cleanups := []string{}
 
 	for {
-		x, ok := readIfExists(t, filepath.Join(dir, CleanupScript))
+		x, ok := tryReading(t, filepath.Join(dir, CleanupScript))
 		if ok {
-			cleanups = append(cleanups, string(x))
+			cleanups = append(cleanups, x)
 		}
 
-		x, ok = readIfExists(t, filepath.Join(dir, PrepareScript))
+		x, ok = tryReading(t, filepath.Join(dir, PrepareScript))
 		if ok {
-			prepares = append(prepares, string(x))
+			prepares = append(prepares, x)
 		}
 
 		if dir == "" || dir == "." {
@@ -417,16 +419,33 @@ func formatOutput(w io.Writer, err error) {
 	}
 }
 
-func readIfExists(t *testing.T, path string) ([]byte, bool) {
-	data, err := os.ReadFile(path)
-	if err == nil {
-		return data, true
+func tryReading(t *testing.T, path string) (string, bool) {
+	info, err := os.Stat(path)
+	if err != nil {
+		if !errors.Is(err, os.ErrNotExist) {
+			t.Errorf("%s: %s", path, err)
+		}
+		return "", false
 	}
 
-	if !errors.Is(err, os.ErrNotExist) {
-		t.Fatalf("%s: %s", path, err)
+	if info.Size() > MaxFileSize {
+		t.Errorf("%s: ignoring, too large: %d", path, info.Size())
+		return "", false
 	}
-	return []byte{}, false
+
+	data, err := os.ReadFile(path)
+	if err != nil {
+		// already checked ErrNotExist above
+		t.Errorf("%s: %s", path, err)
+		return "", false
+	}
+
+	if !utf8.Valid(data) {
+		t.Errorf("%s: not valid utf-8", path)
+		return "", false
+	}
+
+	return string(data), true
 }
 
 func CopyDir(src, dst string, inputs, outputs map[string]bool) error {

From 3ffac800071a397763bddb49e22c1aca4f55573c Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 28 Jan 2025 11:23:44 +0100
Subject: [PATCH 158/247] acc: Use real terraform when CLOUD_ENV is set (#2245)

## Changes
- If CLOUD_ENV is set to do not override with dummy value. This allows
running acceptance tests as integration tests.
- Needed for https://github.com/databricks/cli/pull/2242

## Tests
Manually run the test suite against dogfood. `CLOUD_ENV=aws go test
./acceptance`
---
 acceptance/acceptance_test.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 877c7239d..b4b27f201 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -99,9 +99,6 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	repls.SetPath(tempHomeDir, "$TMPHOME")
 	t.Logf("$TMPHOME=%v", tempHomeDir)
 
-	// Prevent CLI from downloading terraform in each test:
-	t.Setenv("DATABRICKS_TF_EXEC_PATH", tempHomeDir)
-
 	// Make use of uv cache; since we set HomeEnvVar to temporary directory, it is not picked up automatically
 	uvCache := getUVDefaultCacheDir(t)
 	t.Setenv("UV_CACHE_DIR", uvCache)
@@ -119,6 +116,9 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 		homeDir := t.TempDir()
 		// Do not read user's ~/.databrickscfg
 		t.Setenv(env.HomeEnvVar(), homeDir)
+
+		// Prevent CLI from downloading terraform in each test:
+		t.Setenv("DATABRICKS_TF_EXEC_PATH", tempHomeDir)
 	}
 
 	workspaceClient, err := databricks.NewWorkspaceClient()

From 65e4f79dfec84f45689ec3241da62ca3660112e6 Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Tue, 28 Jan 2025 16:24:23 +0530
Subject: [PATCH 159/247] Switch to using `[` from `<` in text replacements
 (#2224)

## Changes
Noticed this when working on
https://github.com/databricks/cli/pull/2221. `<` is a special HTML
character that is encoded during text replacement when using
`AssertEqualTexts`.


## Tests
N/A
---
 .../dbt-sql/output/my_dbt_sql/databricks.yml         |  2 +-
 .../output/my_default_python/databricks.yml          |  2 +-
 .../my_default_python/scratch/exploration.ipynb      |  2 +-
 .../output/my_default_python/src/dlt_pipeline.ipynb  |  6 +++---
 .../output/my_default_python/src/notebook.ipynb      |  4 ++--
 .../default-sql/output/my_default_sql/databricks.yml |  2 +-
 .../output/my_default_sql/scratch/exploration.ipynb  |  2 +-
 .../output/my_jobs_as_code/databricks.yml            |  2 +-
 .../output/my_jobs_as_code/src/notebook.ipynb        |  4 ++--
 .../bundle/testdata/default_python/bundle_deploy.txt |  2 +-
 .../testdata/default_python/bundle_summary.txt       | 12 ++++++------
 libs/testdiff/replacement.go                         |  4 ++--
 libs/testdiff/replacement_test.go                    |  4 ++--
 13 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml
index 1962bc543..cdf3704b9 100644
--- a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml
+++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml
@@ -3,7 +3,7 @@
 # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
 bundle:
   name: my_dbt_sql
-  uuid: <UUID>
+  uuid: [UUID]
 
 include:
   - resources/*.yml
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml b/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml
index 9deca9cf5..3fa777219 100644
--- a/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml
@@ -2,7 +2,7 @@
 # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
 bundle:
   name: my_default_python
-  uuid: <UUID>
+  uuid: [UUID]
 
 include:
   - resources/*.yml
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/scratch/exploration.ipynb b/acceptance/bundle/templates/default-python/output/my_default_python/scratch/exploration.ipynb
index 3b2fef4b4..a12773d4e 100644
--- a/acceptance/bundle/templates/default-python/output/my_default_python/scratch/exploration.ipynb
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/scratch/exploration.ipynb
@@ -20,7 +20,7 @@
       "rowLimit": 10000
      },
      "inputWidgets": {},
-     "nuid": "<UUID>",
+     "nuid": "[UUID]",
      "showTitle": false,
      "title": ""
     }
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/src/dlt_pipeline.ipynb b/acceptance/bundle/templates/default-python/output/my_default_python/src/dlt_pipeline.ipynb
index 36e993af7..8a02183e7 100644
--- a/acceptance/bundle/templates/default-python/output/my_default_python/src/dlt_pipeline.ipynb
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/src/dlt_pipeline.ipynb
@@ -6,7 +6,7 @@
     "application/vnd.databricks.v1+cell": {
      "cellMetadata": {},
      "inputWidgets": {},
-     "nuid": "<UUID>",
+     "nuid": "[UUID]",
      "showTitle": false,
      "title": ""
     }
@@ -24,7 +24,7 @@
     "application/vnd.databricks.v1+cell": {
      "cellMetadata": {},
      "inputWidgets": {},
-     "nuid": "<UUID>",
+     "nuid": "[UUID]",
      "showTitle": false,
      "title": ""
     }
@@ -47,7 +47,7 @@
     "application/vnd.databricks.v1+cell": {
      "cellMetadata": {},
      "inputWidgets": {},
-     "nuid": "<UUID>",
+     "nuid": "[UUID]",
      "showTitle": false,
      "title": ""
     }
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/src/notebook.ipynb b/acceptance/bundle/templates/default-python/output/my_default_python/src/notebook.ipynb
index 0d560443b..472ccb219 100644
--- a/acceptance/bundle/templates/default-python/output/my_default_python/src/notebook.ipynb
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/src/notebook.ipynb
@@ -6,7 +6,7 @@
     "application/vnd.databricks.v1+cell": {
      "cellMetadata": {},
      "inputWidgets": {},
-     "nuid": "<UUID>",
+     "nuid": "[UUID]",
      "showTitle": false,
      "title": ""
     }
@@ -37,7 +37,7 @@
       "rowLimit": 10000
      },
      "inputWidgets": {},
-     "nuid": "<UUID>",
+     "nuid": "[UUID]",
      "showTitle": false,
      "title": ""
     }
diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml b/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml
index ab857287e..16292bc84 100644
--- a/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml
+++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml
@@ -2,7 +2,7 @@
 # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
 bundle:
   name: my_default_sql
-  uuid: <UUID>
+  uuid: [UUID]
 
 include:
   - resources/*.yml
diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/exploration.ipynb b/acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/exploration.ipynb
index c3fd072e5..f3976c1de 100644
--- a/acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/exploration.ipynb
+++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/exploration.ipynb
@@ -7,7 +7,7 @@
     "application/vnd.databricks.v1+cell": {
      "cellMetadata": {},
      "inputWidgets": {},
-     "nuid": "<UUID>",
+     "nuid": "[UUID]",
      "showTitle": false,
      "title": ""
     }
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml
index a1a93d95c..54e69a256 100644
--- a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml
@@ -2,7 +2,7 @@
 # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
 bundle:
   name: my_jobs_as_code
-  uuid: <UUID>
+  uuid: [UUID]
   databricks_cli_version: ">= 0.238.0"
 
 experimental:
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/notebook.ipynb b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/notebook.ipynb
index 9bc3f1560..227c7cc55 100644
--- a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/notebook.ipynb
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/notebook.ipynb
@@ -6,7 +6,7 @@
     "application/vnd.databricks.v1+cell": {
      "cellMetadata": {},
      "inputWidgets": {},
-     "nuid": "<UUID>",
+     "nuid": "[UUID]",
      "showTitle": false,
      "title": ""
     }
@@ -37,7 +37,7 @@
       "rowLimit": 10000
      },
      "inputWidgets": {},
-     "nuid": "<UUID>",
+     "nuid": "[UUID]",
      "showTitle": false,
      "title": ""
     }
diff --git a/integration/bundle/testdata/default_python/bundle_deploy.txt b/integration/bundle/testdata/default_python/bundle_deploy.txt
index eef0b79b3..d7b8cede9 100644
--- a/integration/bundle/testdata/default_python/bundle_deploy.txt
+++ b/integration/bundle/testdata/default_python/bundle_deploy.txt
@@ -1,5 +1,5 @@
 Building project_name_$UNIQUE_PRJ...
-Uploading project_name_$UNIQUE_PRJ-0.0.1+<NUMID>.<NUMID>-py3-none-any.whl...
+Uploading project_name_$UNIQUE_PRJ-0.0.1+[NUMID].[NUMID]-py3-none-any.whl...
 Uploading bundle files to /Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files...
 Deploying resources...
 Updating deployment state...
diff --git a/integration/bundle/testdata/default_python/bundle_summary.txt b/integration/bundle/testdata/default_python/bundle_summary.txt
index 318cd2543..88ccdc496 100644
--- a/integration/bundle/testdata/default_python/bundle_summary.txt
+++ b/integration/bundle/testdata/default_python/bundle_summary.txt
@@ -16,7 +16,7 @@
         "enabled": false
       }
     },
-    "uuid": "<UUID>"
+    "uuid": "[UUID]"
   },
   "include": [
     "resources/project_name_$UNIQUE_PRJ.job.yml",
@@ -74,7 +74,7 @@
           ]
         },
         "format": "MULTI_TASK",
-        "id": "<NUMID>",
+        "id": "[NUMID]",
         "job_clusters": [
           {
             "job_cluster_key": "job_cluster",
@@ -141,7 +141,7 @@
             "unit": "DAYS"
           }
         },
-        "url": "$DATABRICKS_URL/jobs/<NUMID>?o=<NUMID>"
+        "url": "$DATABRICKS_URL/jobs/[NUMID]?o=[NUMID]"
       }
     },
     "pipelines": {
@@ -155,7 +155,7 @@
           "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/state/metadata.json"
         },
         "development": true,
-        "id": "<UUID>",
+        "id": "[UUID]",
         "libraries": [
           {
             "notebook": {
@@ -165,7 +165,7 @@
         ],
         "name": "[dev $USERNAME] project_name_$UNIQUE_PRJ_pipeline",
         "target": "project_name_$UNIQUE_PRJ_dev",
-        "url": "$DATABRICKS_URL/pipelines/<UUID>?o=<NUMID>"
+        "url": "$DATABRICKS_URL/pipelines/[UUID]?o=[NUMID]"
       }
     }
   },
@@ -183,4 +183,4 @@
       "dev": "$USERNAME"
     }
   }
-}
\ No newline at end of file
+}
diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go
index 40e7e72b4..ce5476a57 100644
--- a/libs/testdiff/replacement.go
+++ b/libs/testdiff/replacement.go
@@ -201,12 +201,12 @@ func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam.
 
 func PrepareReplacementsUUID(t testutil.TestingT, r *ReplacementsContext) {
 	t.Helper()
-	r.append(uuidRegex, "<UUID>")
+	r.append(uuidRegex, "[UUID]")
 }
 
 func PrepareReplacementsNumber(t testutil.TestingT, r *ReplacementsContext) {
 	t.Helper()
-	r.append(numIdRegex, "<NUMID>")
+	r.append(numIdRegex, "[NUMID]")
 }
 
 func PrepareReplacementsTemporaryDirectory(t testutil.TestingT, r *ReplacementsContext) {
diff --git a/libs/testdiff/replacement_test.go b/libs/testdiff/replacement_test.go
index de247c03e..1b6c5fe2d 100644
--- a/libs/testdiff/replacement_test.go
+++ b/libs/testdiff/replacement_test.go
@@ -25,7 +25,7 @@ func TestReplacement_UUID(t *testing.T) {
 
 	PrepareReplacementsUUID(t, &repls)
 
-	assert.Equal(t, "<UUID>", repls.Replace("123e4567-e89b-12d3-a456-426614174000"))
+	assert.Equal(t, "[UUID]", repls.Replace("123e4567-e89b-12d3-a456-426614174000"))
 }
 
 func TestReplacement_Number(t *testing.T) {
@@ -34,7 +34,7 @@ func TestReplacement_Number(t *testing.T) {
 	PrepareReplacementsNumber(t, &repls)
 
 	assert.Equal(t, "12", repls.Replace("12"))
-	assert.Equal(t, "<NUMID>", repls.Replace("123"))
+	assert.Equal(t, "[NUMID]", repls.Replace("123"))
 }
 
 func TestReplacement_TemporaryDirectory(t *testing.T) {

From 5971bd5c1ac0997a88f56dd4ccc88acf501e5267 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 28 Jan 2025 15:00:41 +0100
Subject: [PATCH 160/247] acc: Disable git hooks (#2249)

Otherwise hooks from universe and custom hooks run in tests.
---
 acceptance/script.prepare | 1 +
 1 file changed, 1 insertion(+)

diff --git a/acceptance/script.prepare b/acceptance/script.prepare
index b814a1260..ca47cdbff 100644
--- a/acceptance/script.prepare
+++ b/acceptance/script.prepare
@@ -39,6 +39,7 @@ git-repo-init() {
     git config core.autocrlf false
     git config user.name "Tester"
     git config user.email "tester@databricks.com"
+    git config core.hooksPath no-hooks
     git add databricks.yml
     git commit -qm 'Add databricks.yml'
 }

From 025622540809702994aaefdb1e387a6552c00afa Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 28 Jan 2025 15:12:47 +0100
Subject: [PATCH 161/247] acc: Exclude secrets from replacements (#2250)

They should never be printed by CLI anyway.
---
 libs/testdiff/replacement.go | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go
index ce5476a57..7077e611b 100644
--- a/libs/testdiff/replacement.go
+++ b/libs/testdiff/replacement.go
@@ -146,25 +146,16 @@ func PrepareReplacementsWorkspaceClient(t testutil.TestingT, r *ReplacementsCont
 	r.Set(w.Config.ClusterID, "$DATABRICKS_CLUSTER_ID")
 	r.Set(w.Config.WarehouseID, "$DATABRICKS_WAREHOUSE_ID")
 	r.Set(w.Config.ServerlessComputeID, "$DATABRICKS_SERVERLESS_COMPUTE_ID")
-	r.Set(w.Config.MetadataServiceURL, "$DATABRICKS_METADATA_SERVICE_URL")
 	r.Set(w.Config.AccountID, "$DATABRICKS_ACCOUNT_ID")
-	r.Set(w.Config.Token, "$DATABRICKS_TOKEN")
 	r.Set(w.Config.Username, "$DATABRICKS_USERNAME")
-	r.Set(w.Config.Password, "$DATABRICKS_PASSWORD")
 	r.SetPath(w.Config.Profile, "$DATABRICKS_CONFIG_PROFILE")
 	r.Set(w.Config.ConfigFile, "$DATABRICKS_CONFIG_FILE")
 	r.Set(w.Config.GoogleServiceAccount, "$DATABRICKS_GOOGLE_SERVICE_ACCOUNT")
-	r.Set(w.Config.GoogleCredentials, "$GOOGLE_CREDENTIALS")
 	r.Set(w.Config.AzureResourceID, "$DATABRICKS_AZURE_RESOURCE_ID")
-	r.Set(w.Config.AzureClientSecret, "$ARM_CLIENT_SECRET")
-	// r.Set(w.Config.AzureClientID, "$ARM_CLIENT_ID")
 	r.Set(w.Config.AzureClientID, testerName)
 	r.Set(w.Config.AzureTenantID, "$ARM_TENANT_ID")
-	r.Set(w.Config.ActionsIDTokenRequestURL, "$ACTIONS_ID_TOKEN_REQUEST_URL")
-	r.Set(w.Config.ActionsIDTokenRequestToken, "$ACTIONS_ID_TOKEN_REQUEST_TOKEN")
 	r.Set(w.Config.AzureEnvironment, "$ARM_ENVIRONMENT")
 	r.Set(w.Config.ClientID, "$DATABRICKS_CLIENT_ID")
-	r.Set(w.Config.ClientSecret, "$DATABRICKS_CLIENT_SECRET")
 	r.SetPath(w.Config.DatabricksCliPath, "$DATABRICKS_CLI_PATH")
 	// This is set to words like "path" that happen too frequently
 	// r.Set(w.Config.AuthType, "$DATABRICKS_AUTH_TYPE")

From 4ba222ab3632c45e488e88d3c54b6e05cbfe441b Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 28 Jan 2025 15:22:56 +0100
Subject: [PATCH 162/247] Fix env_overrides not to use variables in
 workspace.profile (#2251)

This does not work when this test is run against cloud.

Needed for https://github.com/databricks/cli/pull/2242
---
 acceptance/bundle/variables/env_overrides/databricks.yml | 7 ++++---
 acceptance/bundle/variables/env_overrides/output.txt     | 3 ++-
 acceptance/bundle/variables/env_overrides/script         | 6 +++---
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/acceptance/bundle/variables/env_overrides/databricks.yml b/acceptance/bundle/variables/env_overrides/databricks.yml
index 560513bc3..e5fc7fcc4 100644
--- a/acceptance/bundle/variables/env_overrides/databricks.yml
+++ b/acceptance/bundle/variables/env_overrides/databricks.yml
@@ -18,12 +18,13 @@ variables:
     description: variable with lookup
     lookup:
       cluster_policy: wrong-cluster-policy
+
+  result:
+    default: ${var.a} ${var.b}
+
 bundle:
   name: test bundle
 
-workspace:
-  profile: ${var.a} ${var.b}
-
 targets:
   env-with-single-variable-override:
     variables:
diff --git a/acceptance/bundle/variables/env_overrides/output.txt b/acceptance/bundle/variables/env_overrides/output.txt
index 1ee9ef625..06e6e518b 100644
--- a/acceptance/bundle/variables/env_overrides/output.txt
+++ b/acceptance/bundle/variables/env_overrides/output.txt
@@ -36,5 +36,6 @@ Exit code: 1
   "b": "prod-b",
   "d": "4321",
   "e": "1234",
-  "f": "9876"
+  "f": "9876",
+  "result": "default-a prod-b"
 }
diff --git a/acceptance/bundle/variables/env_overrides/script b/acceptance/bundle/variables/env_overrides/script
index 30919fd8a..3965d1564 100644
--- a/acceptance/bundle/variables/env_overrides/script
+++ b/acceptance/bundle/variables/env_overrides/script
@@ -1,6 +1,6 @@
-trace $CLI bundle validate -t env-with-single-variable-override -o json | jq .workspace.profile
-trace $CLI bundle validate -t env-with-two-variable-overrides -o json | jq .workspace.profile
-trace BUNDLE_VAR_b=env-var-b $CLI bundle validate -t env-with-two-variable-overrides -o json | jq .workspace.profile
+trace $CLI bundle validate -t env-with-single-variable-override -o json | jq .variables.result.value
+trace $CLI bundle validate -t env-with-two-variable-overrides -o json | jq .variables.result.value
+trace BUNDLE_VAR_b=env-var-b $CLI bundle validate -t env-with-two-variable-overrides -o json | jq .variables.result.value
 trace errcode $CLI bundle validate -t env-missing-a-required-variable-assignment
 trace errcode $CLI bundle validate -t env-using-an-undefined-variable
 trace $CLI bundle validate -t env-overrides-lookup -o json | jq '.variables | map_values(.value)'

From 099e9bed0f2250e3dcece80e6e64d8873c75e74d Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Tue, 28 Jan 2025 15:34:44 +0100
Subject: [PATCH 163/247] Upgrade TF provider to 1.64.1 (#2247)

## Changes
- Added support for `no_compute` in Apps
- Added support for `run_as_repl` for job tasks
---
 bundle/internal/tf/codegen/schema/version.go              | 2 +-
 .../internal/tf/schema/data_source_serving_endpoints.go   | 8 ++++----
 bundle/internal/tf/schema/resource_app.go                 | 1 +
 bundle/internal/tf/schema/resource_job.go                 | 2 ++
 bundle/internal/tf/schema/resource_model_serving.go       | 8 ++++----
 bundle/internal/tf/schema/resource_recipient.go           | 1 +
 bundle/internal/tf/schema/root.go                         | 2 +-
 7 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go
index 677b8fc10..393afd6ed 100644
--- a/bundle/internal/tf/codegen/schema/version.go
+++ b/bundle/internal/tf/codegen/schema/version.go
@@ -1,3 +1,3 @@
 package schema
 
-const ProviderVersion = "1.63.0"
+const ProviderVersion = "1.64.1"
diff --git a/bundle/internal/tf/schema/data_source_serving_endpoints.go b/bundle/internal/tf/schema/data_source_serving_endpoints.go
index bdfd778e0..973989216 100644
--- a/bundle/internal/tf/schema/data_source_serving_endpoints.go
+++ b/bundle/internal/tf/schema/data_source_serving_endpoints.go
@@ -3,7 +3,7 @@
 package schema
 
 type DataSourceServingEndpointsEndpointsAiGatewayGuardrailsInputPii struct {
-	Behavior string `json:"behavior"`
+	Behavior string `json:"behavior,omitempty"`
 }
 
 type DataSourceServingEndpointsEndpointsAiGatewayGuardrailsInput struct {
@@ -14,7 +14,7 @@ type DataSourceServingEndpointsEndpointsAiGatewayGuardrailsInput struct {
 }
 
 type DataSourceServingEndpointsEndpointsAiGatewayGuardrailsOutputPii struct {
-	Behavior string `json:"behavior"`
+	Behavior string `json:"behavior,omitempty"`
 }
 
 type DataSourceServingEndpointsEndpointsAiGatewayGuardrailsOutput struct {
@@ -87,8 +87,8 @@ type DataSourceServingEndpointsEndpointsConfigServedEntitiesExternalModelDatabri
 type DataSourceServingEndpointsEndpointsConfigServedEntitiesExternalModelGoogleCloudVertexAiConfig struct {
 	PrivateKey          string `json:"private_key,omitempty"`
 	PrivateKeyPlaintext string `json:"private_key_plaintext,omitempty"`
-	ProjectId           string `json:"project_id,omitempty"`
-	Region              string `json:"region,omitempty"`
+	ProjectId           string `json:"project_id"`
+	Region              string `json:"region"`
 }
 
 type DataSourceServingEndpointsEndpointsConfigServedEntitiesExternalModelOpenaiConfig struct {
diff --git a/bundle/internal/tf/schema/resource_app.go b/bundle/internal/tf/schema/resource_app.go
index 14c93b793..cbce5ab0e 100644
--- a/bundle/internal/tf/schema/resource_app.go
+++ b/bundle/internal/tf/schema/resource_app.go
@@ -91,6 +91,7 @@ type ResourceApp struct {
 	DefaultSourceCodePath    string                        `json:"default_source_code_path,omitempty"`
 	Description              string                        `json:"description,omitempty"`
 	Name                     string                        `json:"name"`
+	NoCompute                bool                          `json:"no_compute,omitempty"`
 	PendingDeployment        *ResourceAppPendingDeployment `json:"pending_deployment,omitempty"`
 	Resources                []ResourceAppResources        `json:"resources,omitempty"`
 	ServicePrincipalClientId string                        `json:"service_principal_client_id,omitempty"`
diff --git a/bundle/internal/tf/schema/resource_job.go b/bundle/internal/tf/schema/resource_job.go
index 63c8aeb7b..da277b5c1 100644
--- a/bundle/internal/tf/schema/resource_job.go
+++ b/bundle/internal/tf/schema/resource_job.go
@@ -904,6 +904,7 @@ type ResourceJobTaskForEachTaskTaskSparkJarTask struct {
 	JarUri        string   `json:"jar_uri,omitempty"`
 	MainClassName string   `json:"main_class_name,omitempty"`
 	Parameters    []string `json:"parameters,omitempty"`
+	RunAsRepl     bool     `json:"run_as_repl,omitempty"`
 }
 
 type ResourceJobTaskForEachTaskTaskSparkPythonTask struct {
@@ -1299,6 +1300,7 @@ type ResourceJobTaskSparkJarTask struct {
 	JarUri        string   `json:"jar_uri,omitempty"`
 	MainClassName string   `json:"main_class_name,omitempty"`
 	Parameters    []string `json:"parameters,omitempty"`
+	RunAsRepl     bool     `json:"run_as_repl,omitempty"`
 }
 
 type ResourceJobTaskSparkPythonTask struct {
diff --git a/bundle/internal/tf/schema/resource_model_serving.go b/bundle/internal/tf/schema/resource_model_serving.go
index 71cf8925d..2025de34c 100644
--- a/bundle/internal/tf/schema/resource_model_serving.go
+++ b/bundle/internal/tf/schema/resource_model_serving.go
@@ -3,7 +3,7 @@
 package schema
 
 type ResourceModelServingAiGatewayGuardrailsInputPii struct {
-	Behavior string `json:"behavior"`
+	Behavior string `json:"behavior,omitempty"`
 }
 
 type ResourceModelServingAiGatewayGuardrailsInput struct {
@@ -14,7 +14,7 @@ type ResourceModelServingAiGatewayGuardrailsInput struct {
 }
 
 type ResourceModelServingAiGatewayGuardrailsOutputPii struct {
-	Behavior string `json:"behavior"`
+	Behavior string `json:"behavior,omitempty"`
 }
 
 type ResourceModelServingAiGatewayGuardrailsOutput struct {
@@ -94,8 +94,8 @@ type ResourceModelServingConfigServedEntitiesExternalModelDatabricksModelServing
 type ResourceModelServingConfigServedEntitiesExternalModelGoogleCloudVertexAiConfig struct {
 	PrivateKey          string `json:"private_key,omitempty"`
 	PrivateKeyPlaintext string `json:"private_key_plaintext,omitempty"`
-	ProjectId           string `json:"project_id,omitempty"`
-	Region              string `json:"region,omitempty"`
+	ProjectId           string `json:"project_id"`
+	Region              string `json:"region"`
 }
 
 type ResourceModelServingConfigServedEntitiesExternalModelOpenaiConfig struct {
diff --git a/bundle/internal/tf/schema/resource_recipient.go b/bundle/internal/tf/schema/resource_recipient.go
index 91de4df76..4c8f2c7e7 100644
--- a/bundle/internal/tf/schema/resource_recipient.go
+++ b/bundle/internal/tf/schema/resource_recipient.go
@@ -29,6 +29,7 @@ type ResourceRecipient struct {
 	CreatedAt                      int                                 `json:"created_at,omitempty"`
 	CreatedBy                      string                              `json:"created_by,omitempty"`
 	DataRecipientGlobalMetastoreId string                              `json:"data_recipient_global_metastore_id,omitempty"`
+	ExpirationTime                 int                                 `json:"expiration_time,omitempty"`
 	Id                             string                              `json:"id,omitempty"`
 	MetastoreId                    string                              `json:"metastore_id,omitempty"`
 	Name                           string                              `json:"name"`
diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go
index 7dd3f9210..2ac852355 100644
--- a/bundle/internal/tf/schema/root.go
+++ b/bundle/internal/tf/schema/root.go
@@ -21,7 +21,7 @@ type Root struct {
 
 const ProviderHost = "registry.terraform.io"
 const ProviderSource = "databricks/databricks"
-const ProviderVersion = "1.63.0"
+const ProviderVersion = "1.64.1"
 
 func NewRoot() *Root {
 	return &Root{

From 413ca5c13471d007edc607a815850f0b31dc32cb Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Tue, 28 Jan 2025 18:17:37 +0100
Subject: [PATCH 164/247] Do not wait for app compute to start on `bundle
 deploy` (#2144)

## Changes
This allows DABs to avoid waiting for the compute to start when app is
initially created as part of "bundle deploy" which significantly
improves deploy time.

Always set no_compute to true for apps

## Tests
Covered by `TestDeployBundleWithApp`, currently fails until TF provider
is upgraded to the version supporting `no_compute` option
---
 bundle/apps/slow_deploy_message.go            | 29 -------------------
 bundle/deploy/terraform/tfdyn/convert_app.go  |  6 ++++
 .../terraform/tfdyn/convert_app_test.go       |  2 ++
 bundle/phases/deploy.go                       |  1 -
 integration/bundle/apps_test.go               | 11 ++++---
 .../bundle/testdata/apps/bundle_deploy.txt    |  1 -
 6 files changed, 13 insertions(+), 37 deletions(-)
 delete mode 100644 bundle/apps/slow_deploy_message.go

diff --git a/bundle/apps/slow_deploy_message.go b/bundle/apps/slow_deploy_message.go
deleted file mode 100644
index 87275980a..000000000
--- a/bundle/apps/slow_deploy_message.go
+++ /dev/null
@@ -1,29 +0,0 @@
-package apps
-
-import (
-	"context"
-
-	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/libs/cmdio"
-	"github.com/databricks/cli/libs/diag"
-)
-
-type slowDeployMessage struct{}
-
-// TODO: needs to be removed when when no_compute option becomes available in TF provider and used in DABs
-// See https://github.com/databricks/cli/pull/2144
-func (v *slowDeployMessage) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-	if len(b.Config.Resources.Apps) > 0 {
-		cmdio.LogString(ctx, "Note: Databricks apps included in this bundle may increase initial deployment time due to compute provisioning.")
-	}
-
-	return nil
-}
-
-func (v *slowDeployMessage) Name() string {
-	return "apps.SlowDeployMessage"
-}
-
-func SlowDeployMessage() bundle.Mutator {
-	return &slowDeployMessage{}
-}
diff --git a/bundle/deploy/terraform/tfdyn/convert_app.go b/bundle/deploy/terraform/tfdyn/convert_app.go
index dcba0809b..b3d599f15 100644
--- a/bundle/deploy/terraform/tfdyn/convert_app.go
+++ b/bundle/deploy/terraform/tfdyn/convert_app.go
@@ -38,6 +38,12 @@ func (appConverter) Convert(ctx context.Context, key string, vin dyn.Value, out
 		return err
 	}
 
+	// We always set no_compute to true as it allows DABs not to wait for app compute to be started when app is created.
+	vout, err = dyn.Set(vout, "no_compute", dyn.V(true))
+	if err != nil {
+		return err
+	}
+
 	// Add the converted resource to the output.
 	out.App[key] = vout.AsAny()
 
diff --git a/bundle/deploy/terraform/tfdyn/convert_app_test.go b/bundle/deploy/terraform/tfdyn/convert_app_test.go
index be8152cc6..cdf56f8ed 100644
--- a/bundle/deploy/terraform/tfdyn/convert_app_test.go
+++ b/bundle/deploy/terraform/tfdyn/convert_app_test.go
@@ -63,6 +63,7 @@ func TestConvertApp(t *testing.T) {
 	assert.Equal(t, map[string]any{
 		"description": "app description",
 		"name":        "app_id",
+		"no_compute":  true,
 		"resources": []any{
 			map[string]any{
 				"name": "job1",
@@ -136,6 +137,7 @@ func TestConvertAppWithNoDescription(t *testing.T) {
 	assert.Equal(t, map[string]any{
 		"name":        "app_id",
 		"description": "", // Due to Apps API always returning a description field, we set it in the output as well to avoid permanent TF drift
+		"no_compute":  true,
 		"resources": []any{
 			map[string]any{
 				"name": "job1",
diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go
index b59ce9f89..c6ec04962 100644
--- a/bundle/phases/deploy.go
+++ b/bundle/phases/deploy.go
@@ -130,7 +130,6 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator {
 	// mutators need informed consent if they are potentially destructive.
 	deployCore := bundle.Defer(
 		bundle.Seq(
-			apps.SlowDeployMessage(),
 			bundle.LogString("Deploying resources..."),
 			terraform.Apply(),
 		),
diff --git a/integration/bundle/apps_test.go b/integration/bundle/apps_test.go
index 23cd784be..01ab52e90 100644
--- a/integration/bundle/apps_test.go
+++ b/integration/bundle/apps_test.go
@@ -18,12 +18,6 @@ import (
 func TestDeployBundleWithApp(t *testing.T) {
 	ctx, wt := acc.WorkspaceTest(t)
 
-	// TODO: should only skip app run when app can be created with no_compute option.
-	if testing.Short() {
-		t.Log("Skip the app creation and run in short mode")
-		return
-	}
-
 	if testutil.GetCloud(t) == testutil.GCP {
 		t.Skip("Skipping test for GCP cloud because /api/2.0/apps is temporarily unavailable there.")
 	}
@@ -106,6 +100,11 @@ env:
   - name: JOB_ID
     value: "%d"`, job.JobId))
 
+	if testing.Short() {
+		t.Log("Skip the app run in short mode")
+		return
+	}
+
 	// Try to run the app
 	_, out := runResourceWithStderr(t, ctx, root, "test_app")
 	require.Contains(t, out, app.Url)
diff --git a/integration/bundle/testdata/apps/bundle_deploy.txt b/integration/bundle/testdata/apps/bundle_deploy.txt
index b077f327d..211164174 100644
--- a/integration/bundle/testdata/apps/bundle_deploy.txt
+++ b/integration/bundle/testdata/apps/bundle_deploy.txt
@@ -1,5 +1,4 @@
 Uploading bundle files to /Workspace/Users/$USERNAME/.bundle/$UNIQUE_PRJ/files...
-Note: Databricks apps included in this bundle may increase initial deployment time due to compute provisioning.
 Deploying resources...
 Updating deployment state...
 Deployment complete!

From 124515e8d2105a3d2ec071dadcb30bf792ba9cad Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Wed, 29 Jan 2025 16:12:21 +0530
Subject: [PATCH 165/247] Move TestServer from acceptance to libs/testserver
 (#2255)

## Changes
Just a move, no changes. As recommended here:
https://github.com/databricks/cli/pull/2226#discussion_r1932152627

## Tests
N/A
---
 acceptance/acceptance_test.go |  3 +-
 acceptance/cmd_server_test.go |  3 +-
 acceptance/server_test.go     | 56 +++----------------------------
 libs/testserver/server.go     | 63 +++++++++++++++++++++++++++++++++++
 4 files changed, 71 insertions(+), 54 deletions(-)
 create mode 100644 libs/testserver/server.go

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index b4b27f201..91ad09e9e 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -20,6 +20,7 @@ import (
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/env"
 	"github.com/databricks/cli/libs/testdiff"
+	"github.com/databricks/cli/libs/testserver"
 	"github.com/databricks/databricks-sdk-go"
 	"github.com/stretchr/testify/require"
 )
@@ -107,7 +108,7 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	cloudEnv := os.Getenv("CLOUD_ENV")
 
 	if cloudEnv == "" {
-		server := StartServer(t)
+		server := testserver.New(t)
 		AddHandlers(server)
 		// Redirect API access to local server:
 		t.Setenv("DATABRICKS_HOST", server.URL)
diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go
index 28feec1bd..3f5a6356e 100644
--- a/acceptance/cmd_server_test.go
+++ b/acceptance/cmd_server_test.go
@@ -8,10 +8,11 @@ import (
 	"testing"
 
 	"github.com/databricks/cli/internal/testcli"
+	"github.com/databricks/cli/libs/testserver"
 	"github.com/stretchr/testify/require"
 )
 
-func StartCmdServer(t *testing.T) *TestServer {
+func StartCmdServer(t *testing.T) *testserver.Server {
 	server := StartServer(t)
 	server.Handle("/", func(r *http.Request) (any, error) {
 		q := r.URL.Query()
diff --git a/acceptance/server_test.go b/acceptance/server_test.go
index dbc55c03f..66de5dcbf 100644
--- a/acceptance/server_test.go
+++ b/acceptance/server_test.go
@@ -1,73 +1,25 @@
 package acceptance_test
 
 import (
-	"encoding/json"
 	"net/http"
-	"net/http/httptest"
 	"testing"
 
+	"github.com/databricks/cli/libs/testserver"
 	"github.com/databricks/databricks-sdk-go/service/catalog"
 	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/iam"
 	"github.com/databricks/databricks-sdk-go/service/workspace"
 )
 
-type TestServer struct {
-	*httptest.Server
-	Mux *http.ServeMux
-}
-
-type HandlerFunc func(r *http.Request) (any, error)
-
-func NewTestServer() *TestServer {
-	mux := http.NewServeMux()
-	server := httptest.NewServer(mux)
-
-	return &TestServer{
-		Server: server,
-		Mux:    mux,
-	}
-}
-
-func (s *TestServer) Handle(pattern string, handler HandlerFunc) {
-	s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) {
-		resp, err := handler(r)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-
-		w.Header().Set("Content-Type", "application/json")
-
-		var respBytes []byte
-
-		respString, ok := resp.(string)
-		if ok {
-			respBytes = []byte(respString)
-		} else {
-			respBytes, err = json.MarshalIndent(resp, "", "    ")
-			if err != nil {
-				http.Error(w, err.Error(), http.StatusInternalServerError)
-				return
-			}
-		}
-
-		if _, err := w.Write(respBytes); err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-	})
-}
-
-func StartServer(t *testing.T) *TestServer {
-	server := NewTestServer()
+func StartServer(t *testing.T) *testserver.Server {
+	server := testserver.New(t)
 	t.Cleanup(func() {
 		server.Close()
 	})
 	return server
 }
 
-func AddHandlers(server *TestServer) {
+func AddHandlers(server *testserver.Server) {
 	server.Handle("GET /api/2.0/policies/clusters/list", func(r *http.Request) (any, error) {
 		return compute.ListPoliciesResponse{
 			Policies: []compute.Policy{
diff --git a/libs/testserver/server.go b/libs/testserver/server.go
new file mode 100644
index 000000000..10269af8f
--- /dev/null
+++ b/libs/testserver/server.go
@@ -0,0 +1,63 @@
+package testserver
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+
+	"github.com/databricks/cli/internal/testutil"
+)
+
+type Server struct {
+	*httptest.Server
+	Mux *http.ServeMux
+
+	t testutil.TestingT
+}
+
+func New(t testutil.TestingT) *Server {
+	mux := http.NewServeMux()
+	server := httptest.NewServer(mux)
+
+	return &Server{
+		Server: server,
+		Mux:    mux,
+		t:      t,
+	}
+}
+
+type HandlerFunc func(req *http.Request) (resp any, err error)
+
+func (s *Server) Close() {
+	s.Server.Close()
+}
+
+func (s *Server) Handle(pattern string, handler HandlerFunc) {
+	s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) {
+		resp, err := handler(r)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+
+		var respBytes []byte
+
+		respString, ok := resp.(string)
+		if ok {
+			respBytes = []byte(respString)
+		} else {
+			respBytes, err = json.MarshalIndent(resp, "", "    ")
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
+		}
+
+		if _, err := w.Write(respBytes); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	})
+}

From 884b5f26ed148c431a0dfea6333bff9b293f8ed1 Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Wed, 29 Jan 2025 16:32:08 +0530
Subject: [PATCH 166/247] Set bundle auth configuration in command context
 (#2195)

## Changes

This change is required to enable tracking execution time telemetry for
bundle commands. In order to track execution time for the command
generally, we need to have the databricks auth configuration available
at this section of the code:


https://github.com/databricks/cli/blob/41bbd89257285707b3c3df9b9e5b92d6bcf8f1d1/cmd/root/root.go#L99

In order to do this we can rely on the `configUsed` context key.

Most commands rely on the `root.MustWorkspaceClient` function which
automatically sets the client config in the `configUsed` context key.
Bundle commands, however, do not do so. They instead store their
workspace clients in the `&bundle.Bundle{}` object.

With this PR, the `configUsed` context key will be set for all `bundle`
commands. Functionally nothing changes.

## Tests
Existing tests. Also manually verified that either
`root.MustConfigureBundle` or `utils.ConfigureBundleWithVariables` is
called for all bundle commands (except `bundle init`) thus ensuring this
context key would be set for all bundle commands.

refs for the functions:
1. `root.MustConfigureBundle`:
https://github.com/databricks/cli/blob/41bbd89257285707b3c3df9b9e5b92d6bcf8f1d1/cmd/root/bundle.go#L88
2. `utils.ConfigureBundleWithVariables`:
https://github.com/databricks/cli/blob/41bbd89257285707b3c3df9b9e5b92d6bcf8f1d1/cmd/bundle/utils/utils.go#L19

---------

Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>
---
 bundle/bundle.go                              | 31 ++++---
 .../mutator/initialize_workspace_client.go    | 26 ------
 bundle/phases/initialize.go                   |  1 -
 cmd/root/auth.go                              |  2 +-
 cmd/root/bundle.go                            | 16 ++++
 cmd/root/bundle_test.go                       | 85 ++++++++-----------
 6 files changed, 69 insertions(+), 92 deletions(-)
 delete mode 100644 bundle/config/mutator/initialize_workspace_client.go

diff --git a/bundle/bundle.go b/bundle/bundle.go
index e715b8b2c..9cb8916f5 100644
--- a/bundle/bundle.go
+++ b/bundle/bundle.go
@@ -72,6 +72,7 @@ type Bundle struct {
 	// It can be initialized on demand after loading the configuration.
 	clientOnce sync.Once
 	client     *databricks.WorkspaceClient
+	clientErr  error
 
 	// Files that are synced to the workspace.file_path
 	Files []fileset.File
@@ -134,23 +135,25 @@ func TryLoad(ctx context.Context) (*Bundle, error) {
 	return Load(ctx, root)
 }
 
-func (b *Bundle) InitializeWorkspaceClient() (*databricks.WorkspaceClient, error) {
-	client, err := b.Config.Workspace.Client()
-	if err != nil {
-		return nil, fmt.Errorf("cannot resolve bundle auth configuration: %w", err)
-	}
-	return client, nil
+func (b *Bundle) WorkspaceClientE() (*databricks.WorkspaceClient, error) {
+	b.clientOnce.Do(func() {
+		var err error
+		b.client, err = b.Config.Workspace.Client()
+		if err != nil {
+			b.clientErr = fmt.Errorf("cannot resolve bundle auth configuration: %w", err)
+		}
+	})
+
+	return b.client, b.clientErr
 }
 
 func (b *Bundle) WorkspaceClient() *databricks.WorkspaceClient {
-	b.clientOnce.Do(func() {
-		var err error
-		b.client, err = b.InitializeWorkspaceClient()
-		if err != nil {
-			panic(err)
-		}
-	})
-	return b.client
+	client, err := b.WorkspaceClientE()
+	if err != nil {
+		panic(err)
+	}
+
+	return client
 }
 
 // SetWorkpaceClient sets the workspace client for this bundle.
diff --git a/bundle/config/mutator/initialize_workspace_client.go b/bundle/config/mutator/initialize_workspace_client.go
deleted file mode 100644
index 5c905f40c..000000000
--- a/bundle/config/mutator/initialize_workspace_client.go
+++ /dev/null
@@ -1,26 +0,0 @@
-package mutator
-
-import (
-	"context"
-
-	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/libs/diag"
-)
-
-type initializeWorkspaceClient struct{}
-
-func InitializeWorkspaceClient() bundle.Mutator {
-	return &initializeWorkspaceClient{}
-}
-
-func (m *initializeWorkspaceClient) Name() string {
-	return "InitializeWorkspaceClient"
-}
-
-// Apply initializes the workspace client for the bundle. We do this here so
-// downstream calls to b.WorkspaceClient() do not panic if there's an error in the
-// auth configuration.
-func (m *initializeWorkspaceClient) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
-	_, err := b.InitializeWorkspaceClient()
-	return diag.FromErr(err)
-}
diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go
index c5b875196..afd6def3f 100644
--- a/bundle/phases/initialize.go
+++ b/bundle/phases/initialize.go
@@ -34,7 +34,6 @@ func Initialize() bundle.Mutator {
 			// If it is an ancestor, this updates all paths to be relative to the sync root path.
 			mutator.SyncInferRoot(),
 
-			mutator.InitializeWorkspaceClient(),
 			mutator.PopulateCurrentUser(),
 			mutator.LoadGitDetails(),
 
diff --git a/cmd/root/auth.go b/cmd/root/auth.go
index 49abfd414..4fcfbb4d8 100644
--- a/cmd/root/auth.go
+++ b/cmd/root/auth.go
@@ -209,7 +209,7 @@ func MustWorkspaceClient(cmd *cobra.Command, args []string) error {
 		if b != nil {
 			ctx = context.WithValue(ctx, &configUsed, b.Config.Workspace.Config())
 			cmd.SetContext(ctx)
-			client, err := b.InitializeWorkspaceClient()
+			client, err := b.WorkspaceClientE()
 			if err != nil {
 				return err
 			}
diff --git a/cmd/root/bundle.go b/cmd/root/bundle.go
index 8b98f2cf2..5842526f3 100644
--- a/cmd/root/bundle.go
+++ b/cmd/root/bundle.go
@@ -81,6 +81,22 @@ func configureBundle(cmd *cobra.Command, b *bundle.Bundle) (*bundle.Bundle, diag
 
 	// Configure the workspace profile if the flag has been set.
 	diags = diags.Extend(configureProfile(cmd, b))
+	if diags.HasError() {
+		return b, diags
+	}
+
+	// Set the auth configuration in the command context. This can be used
+	// downstream to initialize a API client.
+	//
+	// Note that just initializing a workspace client and loading auth configuration
+	// is a fast operation. It does not perform network I/O or invoke processes (for example the Azure CLI).
+	client, err := b.WorkspaceClientE()
+	if err != nil {
+		return b, diags.Extend(diag.FromErr(err))
+	}
+	ctx = context.WithValue(ctx, &configUsed, client.Config)
+	cmd.SetContext(ctx)
+
 	return b, diags
 }
 
diff --git a/cmd/root/bundle_test.go b/cmd/root/bundle_test.go
index 1998b19e6..3517b02e4 100644
--- a/cmd/root/bundle_test.go
+++ b/cmd/root/bundle_test.go
@@ -8,7 +8,6 @@ import (
 	"runtime"
 	"testing"
 
-	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
@@ -38,7 +37,7 @@ func emptyCommand(t *testing.T) *cobra.Command {
 	return cmd
 }
 
-func setupWithHost(t *testing.T, cmd *cobra.Command, host string) *bundle.Bundle {
+func setupWithHost(t *testing.T, cmd *cobra.Command, host string) error {
 	setupDatabricksCfg(t)
 
 	rootPath := t.TempDir()
@@ -51,12 +50,11 @@ workspace:
 	err := os.WriteFile(filepath.Join(rootPath, "databricks.yml"), []byte(contents), 0o644)
 	require.NoError(t, err)
 
-	b, diags := MustConfigureBundle(cmd)
-	require.NoError(t, diags.Error())
-	return b
+	_, diags := MustConfigureBundle(cmd)
+	return diags.Error()
 }
 
-func setupWithProfile(t *testing.T, cmd *cobra.Command, profile string) *bundle.Bundle {
+func setupWithProfile(t *testing.T, cmd *cobra.Command, profile string) error {
 	setupDatabricksCfg(t)
 
 	rootPath := t.TempDir()
@@ -69,29 +67,25 @@ workspace:
 	err := os.WriteFile(filepath.Join(rootPath, "databricks.yml"), []byte(contents), 0o644)
 	require.NoError(t, err)
 
-	b, diags := MustConfigureBundle(cmd)
-	require.NoError(t, diags.Error())
-	return b
+	_, diags := MustConfigureBundle(cmd)
+	return diags.Error()
 }
 
 func TestBundleConfigureDefault(t *testing.T) {
 	testutil.CleanupEnvironment(t)
 
 	cmd := emptyCommand(t)
-	b := setupWithHost(t, cmd, "https://x.com")
-
-	client, err := b.InitializeWorkspaceClient()
+	err := setupWithHost(t, cmd, "https://x.com")
 	require.NoError(t, err)
-	assert.Equal(t, "https://x.com", client.Config.Host)
+
+	assert.Equal(t, "https://x.com", ConfigUsed(cmd.Context()).Host)
 }
 
 func TestBundleConfigureWithMultipleMatches(t *testing.T) {
 	testutil.CleanupEnvironment(t)
 
 	cmd := emptyCommand(t)
-	b := setupWithHost(t, cmd, "https://a.com")
-
-	_, err := b.InitializeWorkspaceClient()
+	err := setupWithHost(t, cmd, "https://a.com")
 	assert.ErrorContains(t, err, "multiple profiles matched: PROFILE-1, PROFILE-2")
 }
 
@@ -101,9 +95,8 @@ func TestBundleConfigureWithNonExistentProfileFlag(t *testing.T) {
 	cmd := emptyCommand(t)
 	err := cmd.Flag("profile").Value.Set("NOEXIST")
 	require.NoError(t, err)
-	b := setupWithHost(t, cmd, "https://x.com")
 
-	_, err = b.InitializeWorkspaceClient()
+	err = setupWithHost(t, cmd, "https://x.com")
 	assert.ErrorContains(t, err, "has no NOEXIST profile configured")
 }
 
@@ -113,9 +106,8 @@ func TestBundleConfigureWithMismatchedProfile(t *testing.T) {
 	cmd := emptyCommand(t)
 	err := cmd.Flag("profile").Value.Set("PROFILE-1")
 	require.NoError(t, err)
-	b := setupWithHost(t, cmd, "https://x.com")
 
-	_, err = b.InitializeWorkspaceClient()
+	err = setupWithHost(t, cmd, "https://x.com")
 	assert.ErrorContains(t, err, "config host mismatch: profile uses host https://a.com, but CLI configured to use https://x.com")
 }
 
@@ -125,12 +117,11 @@ func TestBundleConfigureWithCorrectProfile(t *testing.T) {
 	cmd := emptyCommand(t)
 	err := cmd.Flag("profile").Value.Set("PROFILE-1")
 	require.NoError(t, err)
-	b := setupWithHost(t, cmd, "https://a.com")
+	err = setupWithHost(t, cmd, "https://a.com")
 
-	client, err := b.InitializeWorkspaceClient()
 	require.NoError(t, err)
-	assert.Equal(t, "https://a.com", client.Config.Host)
-	assert.Equal(t, "PROFILE-1", client.Config.Profile)
+	assert.Equal(t, "https://a.com", ConfigUsed(cmd.Context()).Host)
+	assert.Equal(t, "PROFILE-1", ConfigUsed(cmd.Context()).Profile)
 }
 
 func TestBundleConfigureWithMismatchedProfileEnvVariable(t *testing.T) {
@@ -138,9 +129,8 @@ func TestBundleConfigureWithMismatchedProfileEnvVariable(t *testing.T) {
 
 	t.Setenv("DATABRICKS_CONFIG_PROFILE", "PROFILE-1")
 	cmd := emptyCommand(t)
-	b := setupWithHost(t, cmd, "https://x.com")
 
-	_, err := b.InitializeWorkspaceClient()
+	err := setupWithHost(t, cmd, "https://x.com")
 	assert.ErrorContains(t, err, "config host mismatch: profile uses host https://a.com, but CLI configured to use https://x.com")
 }
 
@@ -151,12 +141,11 @@ func TestBundleConfigureWithProfileFlagAndEnvVariable(t *testing.T) {
 	cmd := emptyCommand(t)
 	err := cmd.Flag("profile").Value.Set("PROFILE-1")
 	require.NoError(t, err)
-	b := setupWithHost(t, cmd, "https://a.com")
 
-	client, err := b.InitializeWorkspaceClient()
+	err = setupWithHost(t, cmd, "https://a.com")
 	require.NoError(t, err)
-	assert.Equal(t, "https://a.com", client.Config.Host)
-	assert.Equal(t, "PROFILE-1", client.Config.Profile)
+	assert.Equal(t, "https://a.com", ConfigUsed(cmd.Context()).Host)
+	assert.Equal(t, "PROFILE-1", ConfigUsed(cmd.Context()).Profile)
 }
 
 func TestBundleConfigureProfileDefault(t *testing.T) {
@@ -164,13 +153,12 @@ func TestBundleConfigureProfileDefault(t *testing.T) {
 
 	// The profile in the databricks.yml file is used
 	cmd := emptyCommand(t)
-	b := setupWithProfile(t, cmd, "PROFILE-1")
 
-	client, err := b.InitializeWorkspaceClient()
+	err := setupWithProfile(t, cmd, "PROFILE-1")
 	require.NoError(t, err)
-	assert.Equal(t, "https://a.com", client.Config.Host)
-	assert.Equal(t, "a", client.Config.Token)
-	assert.Equal(t, "PROFILE-1", client.Config.Profile)
+	assert.Equal(t, "https://a.com", ConfigUsed(cmd.Context()).Host)
+	assert.Equal(t, "a", ConfigUsed(cmd.Context()).Token)
+	assert.Equal(t, "PROFILE-1", ConfigUsed(cmd.Context()).Profile)
 }
 
 func TestBundleConfigureProfileFlag(t *testing.T) {
@@ -180,13 +168,12 @@ func TestBundleConfigureProfileFlag(t *testing.T) {
 	cmd := emptyCommand(t)
 	err := cmd.Flag("profile").Value.Set("PROFILE-2")
 	require.NoError(t, err)
-	b := setupWithProfile(t, cmd, "PROFILE-1")
 
-	client, err := b.InitializeWorkspaceClient()
+	err = setupWithProfile(t, cmd, "PROFILE-1")
 	require.NoError(t, err)
-	assert.Equal(t, "https://a.com", client.Config.Host)
-	assert.Equal(t, "b", client.Config.Token)
-	assert.Equal(t, "PROFILE-2", client.Config.Profile)
+	assert.Equal(t, "https://a.com", ConfigUsed(cmd.Context()).Host)
+	assert.Equal(t, "b", ConfigUsed(cmd.Context()).Token)
+	assert.Equal(t, "PROFILE-2", ConfigUsed(cmd.Context()).Profile)
 }
 
 func TestBundleConfigureProfileEnvVariable(t *testing.T) {
@@ -195,13 +182,12 @@ func TestBundleConfigureProfileEnvVariable(t *testing.T) {
 	// The DATABRICKS_CONFIG_PROFILE environment variable takes precedence over the profile in the databricks.yml file
 	t.Setenv("DATABRICKS_CONFIG_PROFILE", "PROFILE-2")
 	cmd := emptyCommand(t)
-	b := setupWithProfile(t, cmd, "PROFILE-1")
 
-	client, err := b.InitializeWorkspaceClient()
+	err := setupWithProfile(t, cmd, "PROFILE-1")
 	require.NoError(t, err)
-	assert.Equal(t, "https://a.com", client.Config.Host)
-	assert.Equal(t, "b", client.Config.Token)
-	assert.Equal(t, "PROFILE-2", client.Config.Profile)
+	assert.Equal(t, "https://a.com", ConfigUsed(cmd.Context()).Host)
+	assert.Equal(t, "b", ConfigUsed(cmd.Context()).Token)
+	assert.Equal(t, "PROFILE-2", ConfigUsed(cmd.Context()).Profile)
 }
 
 func TestBundleConfigureProfileFlagAndEnvVariable(t *testing.T) {
@@ -212,13 +198,12 @@ func TestBundleConfigureProfileFlagAndEnvVariable(t *testing.T) {
 	cmd := emptyCommand(t)
 	err := cmd.Flag("profile").Value.Set("PROFILE-2")
 	require.NoError(t, err)
-	b := setupWithProfile(t, cmd, "PROFILE-1")
 
-	client, err := b.InitializeWorkspaceClient()
+	err = setupWithProfile(t, cmd, "PROFILE-1")
 	require.NoError(t, err)
-	assert.Equal(t, "https://a.com", client.Config.Host)
-	assert.Equal(t, "b", client.Config.Token)
-	assert.Equal(t, "PROFILE-2", client.Config.Profile)
+	assert.Equal(t, "https://a.com", ConfigUsed(cmd.Context()).Host)
+	assert.Equal(t, "b", ConfigUsed(cmd.Context()).Token)
+	assert.Equal(t, "PROFILE-2", ConfigUsed(cmd.Context()).Profile)
 }
 
 func TestTargetFlagFull(t *testing.T) {

From 30f57d3b49aa30ecd9d12bcb416c883035c70f44 Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Wed, 29 Jan 2025 17:19:53 +0530
Subject: [PATCH 167/247] Add protos for bundle telemetry (#2209)

## Changes
These types correspond to the telemetry protobufs defined in universe.

## Tests
No tests are needed since this PR only adds the type bindings.

---------

Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>
---
 libs/telemetry/api.go                       | 31 +++++++++
 libs/telemetry/protos/README.md             |  2 +
 libs/telemetry/protos/bundle_deploy.go      | 77 +++++++++++++++++++++
 libs/telemetry/protos/bundle_init.go        | 37 ++++++++++
 libs/telemetry/protos/databricks_cli_log.go | 35 ++++++++++
 libs/telemetry/protos/enum.go               | 26 +++++++
 libs/telemetry/protos/frontend_log.go       | 22 ++++++
 7 files changed, 230 insertions(+)
 create mode 100644 libs/telemetry/api.go
 create mode 100644 libs/telemetry/protos/README.md
 create mode 100644 libs/telemetry/protos/bundle_deploy.go
 create mode 100644 libs/telemetry/protos/bundle_init.go
 create mode 100644 libs/telemetry/protos/databricks_cli_log.go
 create mode 100644 libs/telemetry/protos/enum.go
 create mode 100644 libs/telemetry/protos/frontend_log.go

diff --git a/libs/telemetry/api.go b/libs/telemetry/api.go
new file mode 100644
index 000000000..4e3f6c861
--- /dev/null
+++ b/libs/telemetry/api.go
@@ -0,0 +1,31 @@
+package telemetry
+
+// RequestBody is the request body type bindings for the /telemetry-ext API endpoint.
+type RequestBody struct {
+	// Timestamp in millis for when the log was uploaded.
+	UploadTime int64 `json:"uploadTime"`
+
+	// DO NOT USE. This is the legacy field for logging in usage logs (not lumberjack).
+	// We keep this around because the API endpoint works only if this field is serialized
+	// to an empty array.
+	Items []string `json:"items"`
+
+	// JSON encoded strings containing the proto logs. Since it's represented as a
+	// string here, the values here end up being double JSON encoded in the final
+	// request body.
+	//
+	// Any logs here will be logged in our lumberjack tables as long as a corresponding
+	// protobuf is defined in universe.
+	ProtoLogs []string `json:"protoLogs"`
+}
+
+// ResponseBody is the response body type bindings for the /telemetry-ext API endpoint.
+type ResponseBody struct {
+	Errors          []LogError `json:"errors"`
+	NumProtoSuccess int64      `json:"numProtoSuccess"`
+}
+
+type LogError struct {
+	Message   string `json:"message"`
+	ErrorType string `json:"errorType"`
+}
diff --git a/libs/telemetry/protos/README.md b/libs/telemetry/protos/README.md
new file mode 100644
index 000000000..7dcc75e17
--- /dev/null
+++ b/libs/telemetry/protos/README.md
@@ -0,0 +1,2 @@
+The types in this package are equivalent to the lumberjack protos defined in Universe.
+You can find all lumberjack protos for the Databricks CLI in the `proto/logs/frontend/databricks_cli` directory.
diff --git a/libs/telemetry/protos/bundle_deploy.go b/libs/telemetry/protos/bundle_deploy.go
new file mode 100644
index 000000000..f3c3a360b
--- /dev/null
+++ b/libs/telemetry/protos/bundle_deploy.go
@@ -0,0 +1,77 @@
+package protos
+
+type BundleDeployEvent struct {
+	// UUID associated with the bundle itself. Set in the `bundle.uuid` field in the bundle configuration.
+	BundleUuid string `json:"bundle_uuid,omitempty"`
+
+	ResourceCount                     int64 `json:"resource_count,omitempty"`
+	ResourceJobCount                  int64 `json:"resource_job_count,omitempty"`
+	ResourcePipelineCount             int64 `json:"resource_pipeline_count,omitempty"`
+	ResourceModelCount                int64 `json:"resource_model_count,omitempty"`
+	ResourceExperimentCount           int64 `json:"resource_experiment_count,omitempty"`
+	ResourceModelServingEndpointCount int64 `json:"resource_model_serving_endpoint_count,omitempty"`
+	ResourceRegisteredModelCount      int64 `json:"resource_registered_model_count,omitempty"`
+	ResourceQualityMonitorCount       int64 `json:"resource_quality_monitor_count,omitempty"`
+	ResourceSchemaCount               int64 `json:"resource_schema_count,omitempty"`
+	ResourceVolumeCount               int64 `json:"resource_volume_count,omitempty"`
+	ResourceClusterCount              int64 `json:"resource_cluster_count,omitempty"`
+	ResourceDashboardCount            int64 `json:"resource_dashboard_count,omitempty"`
+	ResourceAppCount                  int64 `json:"resource_app_count,omitempty"`
+
+	// IDs of resources managed by the bundle. Some resources like volumes or schemas
+	// do not expose a numerical or UUID identifier and are tracked by name. Those
+	// resources are not tracked here since the names are PII.
+	ResourceJobIDs       []string `json:"resource_job_ids,omitempty"`
+	ResourcePipelineIDs  []string `json:"resource_pipeline_ids,omitempty"`
+	ResourceClusterIDs   []string `json:"resource_cluster_ids,omitempty"`
+	ResourceDashboardIDs []string `json:"resource_dashboard_ids,omitempty"`
+
+	Experimental *BundleDeployExperimental `json:"experimental,omitempty"`
+}
+
+// These metrics are experimental and are often added in an adhoc manner. There
+// are no guarantees for these metrics and they maybe removed in the future without
+// any notice.
+type BundleDeployExperimental struct {
+	// Number of configuration files in the bundle.
+	ConfigurationFileCount int64 `json:"configuration_file_count,omitempty"`
+
+	// Size in bytes of the Terraform state file
+	TerraformStateSizeBytes int64 `json:"terraform_state_size_bytes,omitempty"`
+
+	// Number of variables in the bundle
+	VariableCount        int64 `json:"variable_count,omitempty"`
+	ComplexVariableCount int64 `json:"complex_variable_count,omitempty"`
+	LookupVariableCount  int64 `json:"lookup_variable_count,omitempty"`
+
+	// Number of targets in the bundle
+	TargetCount int64 `json:"target_count,omitempty"`
+
+	// Whether a field is set or not. If a configuration field is not present in this
+	// map then it is not tracked by this field.
+	// Keys are the full path of the field in the configuration tree.
+	// Examples: "bundle.terraform.exec_path", "bundle.git.branch" etc.
+	SetFields []BoolMapEntry `json:"set_fields,omitempty"`
+
+	// Values for boolean configuration fields like `experimental.python_wheel_wrapper`
+	// We don't need to define protos to track boolean values and can simply write those
+	// values to this map to track them.
+	BoolValues []BoolMapEntry `json:"bool_values,omitempty"`
+
+	BundleMode BundleMode `json:"bundle_mode,omitempty"`
+
+	WorkspaceArtifactPathType BundleDeployArtifactPathType `json:"workspace_artifact_path_type,omitempty"`
+
+	// Execution time per mutator for a selected subset of mutators.
+	BundleMutatorExecutionTimeMs []IntMapEntry `json:"bundle_mutator_execution_time_ms,omitempty"`
+}
+
+type BoolMapEntry struct {
+	Key   string `json:"key,omitempty"`
+	Value bool   `json:"value,omitempty"`
+}
+
+type IntMapEntry struct {
+	Key   string `json:"key,omitempty"`
+	Value int64  `json:"value,omitempty"`
+}
diff --git a/libs/telemetry/protos/bundle_init.go b/libs/telemetry/protos/bundle_init.go
new file mode 100644
index 000000000..47308a267
--- /dev/null
+++ b/libs/telemetry/protos/bundle_init.go
@@ -0,0 +1,37 @@
+package protos
+
+type BundleInitEvent struct {
+	// UUID associated with the DAB itself. This is serialized into the DAB
+	// when a user runs `databricks bundle init` and all subsequent deployments of
+	// that DAB can then be associated with this init event.
+	BundleUuid string `json:"bundle_uuid,omitempty"`
+
+	// Name of the template initialized when the user ran `databricks bundle init`
+	// This is only populated when the template is a first party template like
+	// mlops-stacks or default-python.
+	TemplateName string `json:"template_name,omitempty"`
+
+	// Arguments used by the user to initialize the template. Only enum
+	// values will be set here by the Databricks CLI.
+	//
+	// We use a generic map representation here because a bundle template's args are
+	// managed in the template itself and maintaining a copy typed schema for it here
+	// will be untenable in the long term.
+	TemplateEnumArgs []BundleInitTemplateEnumArg `json:"template_enum_args,omitempty"`
+}
+
+type BundleInitTemplateEnumArg struct {
+	// Valid key values for the template. These correspond to the keys specified in
+	// the "properties" section of the `databricks_template_schema.json` file.
+	//
+	// Note: `databricks_template_schema.json` contains a JSON schema type specification
+	// for the arguments that the template accepts.
+	Key string `json:"key"`
+
+	// Value that the user set for the field. This is only populated for properties
+	// that have the "enum" field specified in the JSON schema type specification.
+	//
+	// The Databricks CLI ensures that the value here is one of the "enum" values from
+	// the template specification.
+	Value string `json:"value"`
+}
diff --git a/libs/telemetry/protos/databricks_cli_log.go b/libs/telemetry/protos/databricks_cli_log.go
new file mode 100644
index 000000000..9e4e59596
--- /dev/null
+++ b/libs/telemetry/protos/databricks_cli_log.go
@@ -0,0 +1,35 @@
+package protos
+
+type ExecutionContext struct {
+	// UUID generated by the CLI for every CLI command run. This is also set in the HTTP user
+	// agent under the key "cmd-exec-id" and can be used to correlate frontend_log table
+	// with the http_access_log table.
+	CmdExecID string `json:"cmd_exec_id,omitempty"`
+
+	// Version of the Databricks CLI used.
+	Version string `json:"version,omitempty"`
+
+	// Command that was run by the user. Eg: bundle_deploy, fs_cp etc.
+	Command string `json:"command,omitempty"`
+
+	// Lowercase string name for the operating system. Same value
+	// as the one set in `runtime.GOOS` in Golang.
+	OperatingSystem string `json:"operating_system,omitempty"`
+
+	// Version of DBR from which CLI is being run.
+	// Only set when the CLI is being run from a Databricks cluster.
+	DbrVersion string `json:"dbr_version,omitempty"`
+
+	// If true, the CLI is being run from a Databricks notebook / cluster web terminal.
+	FromWebTerminal bool `json:"from_web_terminal,omitempty"`
+
+	// Time taken for the CLI command to execute.
+	ExecutionTimeMs int64 `json:"execution_time_ms,omitempty"`
+
+	// Exit code of the CLI command.
+	ExitCode int64 `json:"exit_code,omitempty"`
+}
+
+type CliTestEvent struct {
+	Name DummyCliEnum `json:"name,omitempty"`
+}
diff --git a/libs/telemetry/protos/enum.go b/libs/telemetry/protos/enum.go
new file mode 100644
index 000000000..7f6780cb6
--- /dev/null
+++ b/libs/telemetry/protos/enum.go
@@ -0,0 +1,26 @@
+package protos
+
+type DummyCliEnum string
+
+const (
+	DummyCliEnumUnspecified DummyCliEnum = "DUMMY_CLI_ENUM_UNSPECIFIED"
+	DummyCliEnumValue1      DummyCliEnum = "VALUE1"
+	DummyCliEnumValue2      DummyCliEnum = "VALUE2"
+	DummyCliEnumValue3      DummyCliEnum = "VALUE3"
+)
+
+type BundleMode string
+
+const (
+	BundleModeUnspecified BundleMode = "TYPE_UNSPECIFIED"
+	BundleModeDevelopment BundleMode = "DEVELOPMENT"
+	BundleModeProduction  BundleMode = "PRODUCTION"
+)
+
+type BundleDeployArtifactPathType string
+
+const (
+	BundleDeployArtifactPathTypeUnspecified BundleDeployArtifactPathType = "TYPE_UNSPECIFIED"
+	BundleDeployArtifactPathTypeWorkspace   BundleDeployArtifactPathType = "WORKSPACE_FILE_SYSTEM"
+	BundleDeployArtifactPathTypeVolume      BundleDeployArtifactPathType = "UC_VOLUME"
+)
diff --git a/libs/telemetry/protos/frontend_log.go b/libs/telemetry/protos/frontend_log.go
new file mode 100644
index 000000000..7e6ab1012
--- /dev/null
+++ b/libs/telemetry/protos/frontend_log.go
@@ -0,0 +1,22 @@
+package protos
+
+// This corresponds to the FrontendLog lumberjack proto in universe.
+// FrontendLog is the top-level struct for any client-side logs at Databricks.
+type FrontendLog struct {
+	// A UUID for the log event generated from the CLI.
+	FrontendLogEventID string `json:"frontend_log_event_id,omitempty"`
+
+	Entry FrontendLogEntry `json:"entry,omitempty"`
+}
+
+type FrontendLogEntry struct {
+	DatabricksCliLog DatabricksCliLog `json:"databricks_cli_log,omitempty"`
+}
+
+type DatabricksCliLog struct {
+	ExecutionContext *ExecutionContext `json:"execution_context,omitempty"`
+
+	CliTestEvent      *CliTestEvent      `json:"cli_test_event,omitempty"`
+	BundleInitEvent   *BundleInitEvent   `json:"bundle_init_event,omitempty"`
+	BundleDeployEvent *BundleDeployEvent `json:"bundle_deploy_event,omitempty"`
+}

From 708c4fbb7af82ff75065e95ca4b50e451c45a315 Mon Sep 17 00:00:00 2001
From: Ilya Kuznetsov <ilya.kuznetsov@databricks.com>
Date: Wed, 29 Jan 2025 13:14:21 +0100
Subject: [PATCH 168/247] Autogenerated documentation for bundle config (#2033)

## Changes

Documentation autogeneration tool. This tool uses same annotations_*.yml
files as in json-schema

Result will go
[there](https://docs.databricks.com/en/dev-tools/bundles/reference.html)
and
[there](https://docs.databricks.com/en/dev-tools/bundles/resources.html#cluster)

## Tests
Manually
---
 Makefile                                      |    5 +-
 bundle/docsgen/README.md                      |   79 +
 bundle/docsgen/main.go                        |  135 +
 bundle/docsgen/markdown.go                    |   99 +
 bundle/docsgen/nodes.go                       |  228 +
 bundle/docsgen/nodes_test.go                  |  120 +
 bundle/docsgen/output/reference.md            | 1336 +++
 bundle/docsgen/output/resources.md            | 8189 +++++++++++++++++
 bundle/docsgen/refs.go                        |   97 +
 bundle/docsgen/renderer.go                    |   51 +
 bundle/docsgen/templates/reference.md         |   10 +
 bundle/docsgen/templates/resources.md         |   70 +
 bundle/internal/annotation/descriptor.go      |   12 +
 bundle/internal/annotation/file.go            |   44 +
 bundle/internal/schema/annotations.go         |   74 +-
 bundle/internal/schema/annotations.yml        |  318 +-
 .../schema/annotations_openapi_overrides.yml  |  355 +
 bundle/internal/schema/annotations_test.go    |    4 +
 bundle/internal/schema/main_test.go           |    5 +-
 bundle/internal/schema/parser.go              |   23 +-
 bundle/schema/jsonschema.json                 |  170 +-
 libs/jsonschema/from_type.go                  |    4 +
 libs/jsonschema/schema.go                     |    4 +
 23 files changed, 11065 insertions(+), 367 deletions(-)
 create mode 100644 bundle/docsgen/README.md
 create mode 100644 bundle/docsgen/main.go
 create mode 100644 bundle/docsgen/markdown.go
 create mode 100644 bundle/docsgen/nodes.go
 create mode 100644 bundle/docsgen/nodes_test.go
 create mode 100644 bundle/docsgen/output/reference.md
 create mode 100644 bundle/docsgen/output/resources.md
 create mode 100644 bundle/docsgen/refs.go
 create mode 100644 bundle/docsgen/renderer.go
 create mode 100644 bundle/docsgen/templates/reference.md
 create mode 100644 bundle/docsgen/templates/resources.md
 create mode 100644 bundle/internal/annotation/descriptor.go
 create mode 100644 bundle/internal/annotation/file.go

diff --git a/Makefile b/Makefile
index 00dadcb0c..d30ccef14 100644
--- a/Makefile
+++ b/Makefile
@@ -48,6 +48,9 @@ vendor:
 schema:
 	go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json
 
+docs:
+	go run ./bundle/docsgen ./bundle/internal/schema ./bundle/docsgen
+
 INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./integration/..." -- -parallel 4 -timeout=2h
 
 integration:
@@ -56,4 +59,4 @@ integration:
 integration-short:
 	$(INTEGRATION) -short
 
-.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover
+.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover docs
diff --git a/bundle/docsgen/README.md b/bundle/docsgen/README.md
new file mode 100644
index 000000000..c8484ab64
--- /dev/null
+++ b/bundle/docsgen/README.md
@@ -0,0 +1,79 @@
+## docs-autogen
+
+1. Install [Golang](https://go.dev/doc/install)
+2. Run `make vendor docs` from the repo
+3. See generated documents in `./bundle/docsgen/output` directory
+4. To change descriptions update content in `./bundle/internal/schema/annotations.yml` or `./bundle/internal/schema/annotations_openapi_overrides.yml` and re-run `make docs`
+
+For simpler usage run it together with copy command to move resulting files to local `docs` repo. Note that it will overwrite any local changes in affected files. Example:
+
+```
+make docs && cp bundle/docgen/output/*.md ../docs/source/dev-tools/bundles
+```
+
+To change intro sections for files update them in `templates/` directory
+
+### Annotation file structure
+
+```yaml
+"<root-type-name>":
+  "<property-name>":
+    description: Description of the property, only plain text is supported
+    markdown_description: Description with markdown support, if defined it will override the value in docs and in JSON-schema
+    markdown_examples: Custom block for any example, in free form, Markdown is supported
+    title: JSON-schema title, not used in docs
+    default: Default value of the property, not used in docs
+    enum: Possible values of enum-type, not used in docs
+```
+
+Descriptions with `PLACEHOLDER` value are not displayed in docs and JSON-schema
+
+All relative links like `[_](/dev-tools/bundles/settings.md#cluster_id)` are kept as is in docs but converted to absolute links in JSON schema
+
+To change description for type itself (not its fields) use `"_"`:
+
+```yaml
+github.com/databricks/cli/bundle/config/resources.Cluster:
+  "_":
+    "markdown_description": |-
+      The cluster resource defines an [all-purpose cluster](/api/workspace/clusters/create).
+```
+
+### Example annotation
+
+```yaml
+github.com/databricks/cli/bundle/config.Bundle:
+  "cluster_id":
+    "description": |-
+      The ID of a cluster to use to run the bundle.
+    "markdown_description": |-
+      The ID of a cluster to use to run the bundle. See [_](/dev-tools/bundles/settings.md#cluster_id).
+  "compute_id":
+    "description": |-
+      PLACEHOLDER
+  "databricks_cli_version":
+    "description": |-
+      The Databricks CLI version to use for the bundle.
+    "markdown_description": |-
+      The Databricks CLI version to use for the bundle. See [_](/dev-tools/bundles/settings.md#databricks_cli_version).
+  "deployment":
+    "description": |-
+      The definition of the bundle deployment
+    "markdown_description": |-
+      The definition of the bundle deployment. For supported attributes, see [_](#deployment) and [_](/dev-tools/bundles/deployment-modes.md).
+  "git":
+    "description": |-
+      The Git version control details that are associated with your bundle.
+    "markdown_description": |-
+      The Git version control details that are associated with your bundle. For supported attributes, see [_](#git) and [_](/dev-tools/bundles/settings.md#git).
+  "name":
+    "description": |-
+      The name of the bundle.
+  "uuid":
+    "description": |-
+      PLACEHOLDER
+```
+
+### TODO
+
+Add file watcher to track changes in the annotation files and re-run `make docs` script automtically
diff --git a/bundle/docsgen/main.go b/bundle/docsgen/main.go
new file mode 100644
index 000000000..ad737feea
--- /dev/null
+++ b/bundle/docsgen/main.go
@@ -0,0 +1,135 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"reflect"
+	"strings"
+
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/internal/annotation"
+	"github.com/databricks/cli/libs/jsonschema"
+)
+
+const (
+	rootFileName      = "reference.md"
+	resourcesFileName = "resources.md"
+)
+
+func main() {
+	if len(os.Args) != 3 {
+		fmt.Println("Usage: go run main.go <annotation-file> <output-file>")
+		os.Exit(1)
+	}
+
+	annotationDir := os.Args[1]
+	docsDir := os.Args[2]
+	outputDir := path.Join(docsDir, "output")
+	templatesDir := path.Join(docsDir, "templates")
+
+	if _, err := os.Stat(outputDir); os.IsNotExist(err) {
+		if err := os.MkdirAll(outputDir, 0o755); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	rootHeader, err := os.ReadFile(path.Join(templatesDir, rootFileName))
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = generateDocs(
+		[]string{path.Join(annotationDir, "annotations.yml")},
+		path.Join(outputDir, rootFileName),
+		reflect.TypeOf(config.Root{}),
+		string(rootHeader),
+	)
+	if err != nil {
+		log.Fatal(err)
+	}
+	resourcesHeader, err := os.ReadFile(path.Join(templatesDir, resourcesFileName))
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = generateDocs(
+		[]string{path.Join(annotationDir, "annotations_openapi.yml"), path.Join(annotationDir, "annotations_openapi_overrides.yml"), path.Join(annotationDir, "annotations.yml")},
+		path.Join(outputDir, resourcesFileName),
+		reflect.TypeOf(config.Resources{}),
+		string(resourcesHeader),
+	)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func generateDocs(inputPaths []string, outputPath string, rootType reflect.Type, header string) error {
+	annotations, err := annotation.LoadAndMerge(inputPaths)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// schemas is used to resolve references to schemas
+	schemas := map[string]*jsonschema.Schema{}
+	// ownFields is used to track fields that are defined in the annotation file and should be included in the docs page
+	ownFields := map[string]bool{}
+
+	s, err := jsonschema.FromType(rootType, []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{
+		func(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
+			_, isOwnField := annotations[jsonschema.TypePath(typ)]
+			if isOwnField {
+				ownFields[jsonschema.TypePath(typ)] = true
+			}
+
+			refPath := getPath(typ)
+			shouldHandle := strings.HasPrefix(refPath, "github.com")
+			if !shouldHandle {
+				schemas[jsonschema.TypePath(typ)] = &s
+				return s
+			}
+
+			a := annotations[refPath]
+			if a == nil {
+				a = map[string]annotation.Descriptor{}
+			}
+
+			rootTypeAnnotation, ok := a["_"]
+			if ok {
+				assignAnnotation(&s, rootTypeAnnotation)
+			}
+
+			for k, v := range s.Properties {
+				assignAnnotation(v, a[k])
+			}
+
+			schemas[jsonschema.TypePath(typ)] = &s
+			return s
+		},
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	nodes := buildNodes(s, schemas, ownFields)
+	err = buildMarkdown(nodes, outputPath, header)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return nil
+}
+
+func getPath(typ reflect.Type) string {
+	return typ.PkgPath() + "." + typ.Name()
+}
+
+func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) {
+	if a.Description != "" && a.Description != annotation.Placeholder {
+		s.Description = a.Description
+	}
+	if a.MarkdownDescription != "" {
+		s.MarkdownDescription = a.MarkdownDescription
+	}
+	if a.MarkdownExamples != "" {
+		s.Examples = []any{a.MarkdownExamples}
+	}
+}
diff --git a/bundle/docsgen/markdown.go b/bundle/docsgen/markdown.go
new file mode 100644
index 000000000..6e3b42b65
--- /dev/null
+++ b/bundle/docsgen/markdown.go
@@ -0,0 +1,99 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"strings"
+)
+
+func buildMarkdown(nodes []rootNode, outputFile, header string) error {
+	m := newMardownRenderer()
+	m = m.PlainText(header)
+	for _, node := range nodes {
+		m = m.LF()
+		if node.TopLevel {
+			m = m.H2(node.Title)
+		} else {
+			m = m.H3(node.Title)
+		}
+		m = m.LF()
+
+		if node.Type != "" {
+			m = m.PlainText(fmt.Sprintf("**`Type: %s`**", node.Type))
+			m = m.LF()
+		}
+		m = m.PlainText(node.Description)
+		m = m.LF()
+
+		if len(node.ObjectKeyAttributes) > 0 {
+			n := pickLastWord(node.Title)
+			n = removePluralForm(n)
+			m = m.CodeBlocks("yaml", fmt.Sprintf("%ss:\n  <%s-name>:\n    <%s-field-name>: <%s-field-value>", n, n, n, n))
+			m = m.LF()
+			m = buildAttributeTable(m, node.ObjectKeyAttributes)
+		} else if len(node.ArrayItemAttributes) > 0 {
+			m = m.LF()
+			m = buildAttributeTable(m, node.ArrayItemAttributes)
+		} else if len(node.Attributes) > 0 {
+			m = m.LF()
+			m = buildAttributeTable(m, node.Attributes)
+		}
+
+		if node.Example != "" {
+			m = m.LF()
+			m = m.PlainText("**Example**")
+			m = m.LF()
+			m = m.PlainText(node.Example)
+		}
+	}
+
+	f, err := os.Create(outputFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	_, err = f.WriteString(m.String())
+	if err != nil {
+		log.Fatal(err)
+	}
+	return f.Close()
+}
+
+func pickLastWord(s string) string {
+	words := strings.Split(s, ".")
+	return words[len(words)-1]
+}
+
+// Build a custom table which we use in Databricks website
+func buildAttributeTable(m *markdownRenderer, attributes []attributeNode) *markdownRenderer {
+	m = m.LF()
+	m = m.PlainText(".. list-table::")
+	m = m.PlainText("   :header-rows: 1")
+	m = m.LF()
+
+	m = m.PlainText("   * - Key")
+	m = m.PlainText("     - Type")
+	m = m.PlainText("     - Description")
+	m = m.LF()
+
+	for _, a := range attributes {
+		m = m.PlainText("   * - " + fmt.Sprintf("`%s`", a.Title))
+		m = m.PlainText("     - " + a.Type)
+		m = m.PlainText("     - " + formatDescription(a))
+		m = m.LF()
+	}
+	return m
+}
+
+func formatDescription(a attributeNode) string {
+	s := strings.ReplaceAll(a.Description, "\n", " ")
+	if a.Link != "" {
+		if strings.HasSuffix(s, ".") {
+			s += " "
+		} else if s != "" {
+			s += ". "
+		}
+		s += fmt.Sprintf("See [_](#%s).", a.Link)
+	}
+	return s
+}
diff --git a/bundle/docsgen/nodes.go b/bundle/docsgen/nodes.go
new file mode 100644
index 000000000..68ed86450
--- /dev/null
+++ b/bundle/docsgen/nodes.go
@@ -0,0 +1,228 @@
+package main
+
+import (
+	"sort"
+	"strings"
+
+	"github.com/databricks/cli/libs/jsonschema"
+)
+
+// rootNode is an intermediate representation of resolved JSON-schema item that is used to generate documentation
+// Every schema node goes follows this conversion `JSON-schema -> rootNode -> markdown text`
+type rootNode struct {
+	Title               string
+	Description         string
+	Attributes          []attributeNode
+	Example             string
+	ObjectKeyAttributes []attributeNode
+	ArrayItemAttributes []attributeNode
+	TopLevel            bool
+	Type                string
+}
+
+type attributeNode struct {
+	Title       string
+	Type        string
+	Description string
+	Link        string
+}
+
+type rootProp struct {
+	// k is the name of the property
+	k string
+	// v is the corresponding json-schema node
+	v *jsonschema.Schema
+	// topLevel is true only for direct properties of the schema of root type (e.g. config.Root or config.Resources)
+	// Example: config.Root has .
+	topLevel bool
+	// circular indicates if property was added by recursive type, e.g. task.for_each_task.task.for_each_task
+	// These entries don't expand further and don't add any new nodes from their properties
+	circular bool
+}
+
+const MapType = "Map"
+
+// buildNodes converts JSON-schema to a flat list of rootNode items that are then used to generate markdown documentation
+// It recursively traverses the schema expanding the resulting list with new items for every properties of nodes `object` and `array` type
+func buildNodes(s jsonschema.Schema, refs map[string]*jsonschema.Schema, ownFields map[string]bool) []rootNode {
+	rootProps := []rootProp{}
+	for k, v := range s.Properties {
+		rootProps = append(rootProps, rootProp{k, v, true, false})
+	}
+	nodes := make([]rootNode, 0, len(rootProps))
+	visited := make(map[string]bool)
+
+	for i := 0; i < len(rootProps); i++ {
+		item := rootProps[i]
+		k := item.k
+		v := item.v
+
+		if visited[k] {
+			continue
+		}
+		visited[k] = true
+
+		v = resolveRefs(v, refs)
+		node := rootNode{
+			Title:       k,
+			Description: getDescription(v, item.topLevel),
+			TopLevel:    item.topLevel,
+			Example:     getExample(v),
+			Type:        getHumanReadableType(v.Type),
+		}
+
+		hasProperties := len(v.Properties) > 0
+		if hasProperties {
+			node.Attributes = getAttributes(v.Properties, refs, ownFields, k, item.circular)
+		}
+
+		mapValueType := getMapValueType(v, refs)
+		if mapValueType != nil {
+			d := getDescription(mapValueType, true)
+			if d != "" {
+				node.Description = d
+			}
+			if node.Example == "" {
+				node.Example = getExample(mapValueType)
+			}
+			node.ObjectKeyAttributes = getAttributes(mapValueType.Properties, refs, ownFields, getMapKeyPrefix(k), item.circular)
+		}
+
+		arrayItemType := resolveRefs(v.Items, refs)
+		if arrayItemType != nil {
+			node.ArrayItemAttributes = getAttributes(arrayItemType.Properties, refs, ownFields, k, item.circular)
+		}
+
+		nodes = append(nodes, node)
+
+		// Whether we should add new root props from the children of the current JSON-schema node to include their definitions to this document
+		shouldAddNewProps := !item.circular
+		if shouldAddNewProps {
+			newProps := []rootProp{}
+			// Adds node with definition for the properties. Example:
+			// bundle:
+			//  prop-name: <value>
+			if hasProperties {
+				newProps = append(newProps, extractNodes(k, v.Properties, refs, ownFields)...)
+			}
+
+			// Adds node with definition for the type of array item. Example:
+			// permissions:
+			//  - <item>
+			if arrayItemType != nil {
+				newProps = append(newProps, extractNodes(k, arrayItemType.Properties, refs, ownFields)...)
+			}
+			// Adds node with definition for the type of the Map value. Example:
+			// targets:
+			//   <key>: <value>
+			if mapValueType != nil {
+				newProps = append(newProps, extractNodes(getMapKeyPrefix(k), mapValueType.Properties, refs, ownFields)...)
+			}
+
+			rootProps = append(rootProps, newProps...)
+		}
+	}
+
+	sort.Slice(nodes, func(i, j int) bool {
+		return nodes[i].Title < nodes[j].Title
+	})
+	return nodes
+}
+
+func getMapValueType(v *jsonschema.Schema, refs map[string]*jsonschema.Schema) *jsonschema.Schema {
+	additionalProps, ok := v.AdditionalProperties.(*jsonschema.Schema)
+	if ok {
+		return resolveRefs(additionalProps, refs)
+	}
+	return nil
+}
+
+func getMapKeyPrefix(s string) string {
+	return s + ".<name>"
+}
+
+func removePluralForm(s string) string {
+	if strings.HasSuffix(s, "s") {
+		return strings.TrimSuffix(s, "s")
+	}
+	return s
+}
+
+func getHumanReadableType(t jsonschema.Type) string {
+	typesMapping := map[string]string{
+		"string":  "String",
+		"integer": "Integer",
+		"boolean": "Boolean",
+		"array":   "Sequence",
+		"object":  "Map",
+	}
+	return typesMapping[string(t)]
+}
+
+func getAttributes(props, refs map[string]*jsonschema.Schema, ownFields map[string]bool, prefix string, circular bool) []attributeNode {
+	attributes := []attributeNode{}
+	for k, v := range props {
+		v = resolveRefs(v, refs)
+		typeString := getHumanReadableType(v.Type)
+		if typeString == "" {
+			typeString = "Any"
+		}
+		var reference string
+		if isReferenceType(v, refs, ownFields) && !circular {
+			reference = prefix + "." + k
+		}
+		attributes = append(attributes, attributeNode{
+			Title:       k,
+			Type:        typeString,
+			Description: getDescription(v, true),
+			Link:        reference,
+		})
+	}
+	sort.Slice(attributes, func(i, j int) bool {
+		return attributes[i].Title < attributes[j].Title
+	})
+	return attributes
+}
+
+func getDescription(s *jsonschema.Schema, allowMarkdown bool) string {
+	if allowMarkdown && s.MarkdownDescription != "" {
+		return s.MarkdownDescription
+	}
+	return s.Description
+}
+
+func shouldExtract(ref string, ownFields map[string]bool) bool {
+	if i := strings.Index(ref, "github.com"); i >= 0 {
+		ref = ref[i:]
+	}
+	_, isCustomField := ownFields[ref]
+	return isCustomField
+}
+
+// extractNodes returns a list of rootProp items for all properties of the json-schema node that should be extracted based on context
+// E.g. we extract all propert
+func extractNodes(prefix string, props, refs map[string]*jsonschema.Schema, ownFields map[string]bool) []rootProp {
+	nodes := []rootProp{}
+	for k, v := range props {
+		if v.Reference != nil && !shouldExtract(*v.Reference, ownFields) {
+			continue
+		}
+		v = resolveRefs(v, refs)
+		if v.Type == "object" || v.Type == "array" {
+			nodes = append(nodes, rootProp{prefix + "." + k, v, false, isCycleField(k)})
+		}
+	}
+	return nodes
+}
+
+func isCycleField(field string) bool {
+	return field == "for_each_task"
+}
+
+func getExample(v *jsonschema.Schema) string {
+	examples := v.Examples
+	if len(examples) == 0 {
+		return ""
+	}
+	return examples[0].(string)
+}
diff --git a/bundle/docsgen/nodes_test.go b/bundle/docsgen/nodes_test.go
new file mode 100644
index 000000000..bdb2ce9db
--- /dev/null
+++ b/bundle/docsgen/nodes_test.go
@@ -0,0 +1,120 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/databricks/cli/libs/jsonschema"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBuildNodes_ChildExpansion(t *testing.T) {
+	tests := []struct {
+		name      string
+		schema    jsonschema.Schema
+		refs      map[string]*jsonschema.Schema
+		ownFields map[string]bool
+		wantNodes []rootNode
+	}{
+		{
+			name: "array expansion",
+			schema: jsonschema.Schema{
+				Type: "object",
+				Properties: map[string]*jsonschema.Schema{
+					"list": {
+						Type: "array",
+						Items: &jsonschema.Schema{
+							Type: "object",
+							Properties: map[string]*jsonschema.Schema{
+								"listSub": {Reference: strPtr("#/$defs/github.com/listSub")},
+							},
+						},
+					},
+				},
+			},
+			refs: map[string]*jsonschema.Schema{
+				"github.com/listSub": {Type: "array", Items: &jsonschema.Schema{Type: "object", Properties: map[string]*jsonschema.Schema{"subField": {Type: "string"}}}},
+			},
+			ownFields: map[string]bool{"github.com/listSub": true},
+			wantNodes: []rootNode{
+				{
+					Title:    "list",
+					TopLevel: true,
+					Type:     "Sequence",
+					ArrayItemAttributes: []attributeNode{
+						{Title: "listSub", Type: "Sequence", Link: "list.listSub"},
+					},
+				},
+				{
+					Title: "list.listSub",
+					Type:  "Sequence",
+					ArrayItemAttributes: []attributeNode{
+						{Title: "subField", Type: "String"},
+					},
+				},
+			},
+		},
+		{
+			name: "map expansion",
+			schema: jsonschema.Schema{
+				Type: "object",
+				Properties: map[string]*jsonschema.Schema{
+					"myMap": {
+						Type: "object",
+						AdditionalProperties: &jsonschema.Schema{
+							Reference: strPtr("#/$defs/github.com/myMap"),
+							Properties: map[string]*jsonschema.Schema{
+								"mapSub": {Type: "object", Reference: strPtr("#/$defs/github.com/mapSub")},
+							},
+						},
+					},
+				},
+			},
+			refs: map[string]*jsonschema.Schema{
+				"github.com/myMap": {
+					Type: "object",
+					Properties: map[string]*jsonschema.Schema{
+						"mapSub": {Type: "boolean", Reference: strPtr("#/$defs/github.com/mapSub")},
+					},
+				},
+				"github.com/mapSub": {
+					Type: "object",
+					Properties: map[string]*jsonschema.Schema{
+						"deepSub": {Type: "boolean"},
+					},
+				},
+			},
+			ownFields: map[string]bool{
+				"github.com/myMap":  true,
+				"github.com/mapSub": true,
+			},
+			wantNodes: []rootNode{
+				{
+					Title:    "myMap",
+					TopLevel: true,
+					Type:     "Map",
+					ObjectKeyAttributes: []attributeNode{
+						{Title: "mapSub", Type: "Map", Link: "myMap.<name>.mapSub"},
+					},
+				},
+				{
+					Title: "myMap.<name>.mapSub",
+					Type:  "Map",
+					Attributes: []attributeNode{
+						{Title: "deepSub", Type: "Boolean"},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := buildNodes(tt.schema, tt.refs, tt.ownFields)
+			assert.Equal(t, tt.wantNodes, got)
+		})
+	}
+}
+
+func strPtr(s string) *string {
+	return &s
+}
diff --git a/bundle/docsgen/output/reference.md b/bundle/docsgen/output/reference.md
new file mode 100644
index 000000000..a2241d017
--- /dev/null
+++ b/bundle/docsgen/output/reference.md
@@ -0,0 +1,1336 @@
+<!-- DO NOT EDIT. This file is autogenerated with https://github.com/databricks/cli -->
+---
+description: Configuration reference for databricks.yml
+---
+
+# Configuration reference
+
+This article provides reference for keys supported by <DABS> configuration (YAML). See [_](/dev-tools/bundles/index.md).
+
+For complete bundle examples, see [_](/dev-tools/bundles/resource-examples.md) and the [bundle-examples GitHub repository](https://github.com/databricks/bundle-examples).
+
+  
+## artifacts
+  
+**`Type: Map`**
+  
+Defines the attributes to build artifacts, where each key is the name of the artifact, and the value is a Map that defines the artifact build settings. For information about the `artifacts` mapping, see [_](/dev-tools/bundles/settings.md#artifacts).
+
+Artifact settings defined in the top level of the bundle configuration can be overridden in the `targets` mapping. See [_](/dev-tools/bundles/artifact-overrides.md).
+  
+```yaml
+artifacts:
+  <artifact-name>:
+    <artifact-field-name>: <artifact-field-value>
+```
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `build`
+     - String
+     - An optional set of non-default build commands to run locally before deployment.
+  
+   * - `executable`
+     - String
+     - The executable type. Valid values are `bash`, `sh`, and `cmd`.
+  
+   * - `files`
+     - Sequence
+     - The source files for the artifact. See [_](#artifacts.<name>.files).
+  
+   * - `path`
+     - String
+     - The location where the built artifact will be saved.
+  
+   * - `type`
+     - String
+     - Required. The type of the artifact. Valid values are `whl`.
+  
+  
+**Example**
+  
+```yaml
+artifacts:
+  default:
+    type: whl
+    build: poetry build
+    path: .
+```
+  
+### artifacts.<name>.files
+  
+**`Type: Sequence`**
+  
+The source files for the artifact.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `source`
+     - String
+     - Required. The path of the files used to build the artifact.
+  
+  
+## bundle
+  
+**`Type: Map`**
+  
+The bundle attributes when deploying to this target,
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `cluster_id`
+     - String
+     - The ID of a cluster to use to run the bundle. See [_](/dev-tools/bundles/settings.md#cluster_id).
+  
+   * - `compute_id`
+     - String
+     - 
+  
+   * - `databricks_cli_version`
+     - String
+     - The Databricks CLI version to use for the bundle. See [_](/dev-tools/bundles/settings.md#databricks_cli_version).
+  
+   * - `deployment`
+     - Map
+     - The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). See [_](#bundle.deployment).
+  
+   * - `git`
+     - Map
+     - The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). See [_](#bundle.git).
+  
+   * - `name`
+     - String
+     - The name of the bundle.
+  
+   * - `uuid`
+     - String
+     - Reserved. A Universally Unique Identifier (UUID) for the bundle that uniquely identifies the bundle in internal Databricks systems. This is generated when a bundle project is initialized using a Databricks template (using the `databricks bundle init` command).
+  
+  
+### bundle.deployment
+  
+**`Type: Map`**
+  
+The definition of the bundle deployment
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `fail_on_active_runs`
+     - Boolean
+     - Whether to fail on active runs. If this is set to true a deployment that is running can be interrupted.
+  
+   * - `lock`
+     - Map
+     - The deployment lock attributes. See [_](#bundle.deployment.lock).
+  
+  
+### bundle.deployment.lock
+  
+**`Type: Map`**
+  
+The deployment lock attributes.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `enabled`
+     - Boolean
+     - Whether this lock is enabled.
+  
+   * - `force`
+     - Boolean
+     - Whether to force this lock if it is enabled.
+  
+  
+### bundle.git
+  
+**`Type: Map`**
+  
+The Git version control details that are associated with your bundle.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `branch`
+     - String
+     - The Git branch name. See [_](/dev-tools/bundles/settings.md#git).
+  
+   * - `origin_url`
+     - String
+     - The origin URL of the repository. See [_](/dev-tools/bundles/settings.md#git).
+  
+  
+## experimental
+  
+**`Type: Map`**
+  
+Defines attributes for experimental features.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `pydabs`
+     - Map
+     - The PyDABs configuration. See [_](#experimental.pydabs).
+  
+   * - `python`
+     - Map
+     - Configures loading of Python code defined with 'databricks-bundles' package. See [_](#experimental.python).
+  
+   * - `python_wheel_wrapper`
+     - Boolean
+     - Whether to use a Python wheel wrapper.
+  
+   * - `scripts`
+     - Map
+     - The commands to run.
+  
+   * - `use_legacy_run_as`
+     - Boolean
+     - Whether to use the legacy run_as behavior.
+  
+  
+### experimental.pydabs
+  
+**`Type: Map`**
+  
+The PyDABs configuration.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `enabled`
+     - Boolean
+     - Whether or not PyDABs (Private Preview) is enabled
+  
+   * - `import`
+     - Sequence
+     - The PyDABs project to import to discover resources, resource generator and mutators
+  
+   * - `venv_path`
+     - String
+     - The Python virtual environment path
+  
+  
+### experimental.python
+  
+**`Type: Map`**
+  
+Configures loading of Python code defined with 'databricks-bundles' package.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `mutators`
+     - Sequence
+     - Mutators contains a list of fully qualified function paths to mutator functions.  Example: ["my_project.mutators:add_default_cluster"]
+  
+   * - `resources`
+     - Sequence
+     - Resources contains a list of fully qualified function paths to load resources defined in Python code.  Example: ["my_project.resources:load_resources"]
+  
+   * - `venv_path`
+     - String
+     - VEnvPath is path to the virtual environment.  If enabled, Python code will execute within this environment. If disabled, it defaults to using the Python interpreter available in the current shell.
+  
+  
+## include
+  
+**`Type: Sequence`**
+  
+Specifies a list of path globs that contain configuration files to include within the bundle. See [_](/dev-tools/bundles/settings.md#include)
+  
+  
+## permissions
+  
+**`Type: Sequence`**
+  
+A Sequence that defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle, where each item in the sequence is a permission for a specific entity.
+
+See [_](/dev-tools/bundles/settings.md#permissions) and [_](/dev-tools/bundles/permissions.md).
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `group_name`
+     - String
+     - The name of the group that has the permission set in level.
+  
+   * - `level`
+     - String
+     - The allowed permission for user, group, service principal defined for this permission.
+  
+   * - `service_principal_name`
+     - String
+     - The name of the service principal that has the permission set in level.
+  
+   * - `user_name`
+     - String
+     - The name of the user that has the permission set in level.
+  
+  
+**Example**
+  
+```yaml
+permissions:
+  - level: CAN_VIEW
+    group_name: test-group
+  - level: CAN_MANAGE
+    user_name: someone@example.com
+  - level: CAN_RUN
+    service_principal_name: 123456-abcdef
+```
+  
+## presets
+  
+**`Type: Map`**
+  
+Defines bundle deployment presets. See [_](/dev-tools/bundles/deployment-modes.md#presets).
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `jobs_max_concurrent_runs`
+     - Integer
+     - The maximum concurrent runs for a job.
+  
+   * - `name_prefix`
+     - String
+     - The prefix for job runs of the bundle.
+  
+   * - `pipelines_development`
+     - Boolean
+     - Whether pipeline deployments should be locked in development mode.
+  
+   * - `source_linked_deployment`
+     - Boolean
+     - Whether to link the deployment to the bundle source.
+  
+   * - `tags`
+     - Map
+     - The tags for the bundle deployment.
+  
+   * - `trigger_pause_status`
+     - String
+     - A pause status to apply to all job triggers and schedules. Valid values are PAUSED or UNPAUSED.
+  
+  
+## resources
+  
+**`Type: Map`**
+  
+A Map that defines the resources for the bundle, where each key is the name of the resource, and the value is a Map that defines the resource. For more information about <DABS> supported resources, and resource definition reference, see [_](/dev-tools/bundles/resources.md).
+
+```yaml
+resources:
+  <resource-type>:
+    <resource-name>:
+      <resource-field-name>: <resource-field-value>
+```
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `apps`
+     - Map
+     - 
+  
+   * - `clusters`
+     - Map
+     - The cluster definitions for the bundle, where each key is the name of a cluster. See [_](/dev-tools/bundles/resources.md#clusters)
+  
+   * - `dashboards`
+     - Map
+     - The dashboard definitions for the bundle, where each key is the name of the dashboard. See [_](/dev-tools/bundles/resources.md#dashboards)
+  
+   * - `experiments`
+     - Map
+     - The experiment definitions for the bundle, where each key is the name of the experiment. See [_](/dev-tools/bundles/resources.md#experiments)
+  
+   * - `jobs`
+     - Map
+     - The job definitions for the bundle, where each key is the name of the job. See [_](/dev-tools/bundles/resources.md#jobs)
+  
+   * - `model_serving_endpoints`
+     - Map
+     - The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint. See [_](/dev-tools/bundles/resources.md#model_serving_endpoints)
+  
+   * - `models`
+     - Map
+     - The model definitions for the bundle, where each key is the name of the model. See [_](/dev-tools/bundles/resources.md#models)
+  
+   * - `pipelines`
+     - Map
+     - The pipeline definitions for the bundle, where each key is the name of the pipeline. See [_](/dev-tools/bundles/resources.md#pipelines)
+  
+   * - `quality_monitors`
+     - Map
+     - The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [_](/dev-tools/bundles/resources.md#quality_monitors)
+  
+   * - `registered_models`
+     - Map
+     - The registered model definitions for the bundle, where each key is the name of the <UC> registered model. See [_](/dev-tools/bundles/resources.md#registered_models)
+  
+   * - `schemas`
+     - Map
+     - The schema definitions for the bundle, where each key is the name of the schema. See [_](/dev-tools/bundles/resources.md#schemas)
+  
+   * - `volumes`
+     - Map
+     - The volume definitions for the bundle, where each key is the name of the volume. See [_](/dev-tools/bundles/resources.md#volumes)
+  
+  
+## run_as
+  
+**`Type: Map`**
+  
+The identity to use when running <DABS> workflows. See [_](/dev-tools/bundles/run-as.md).
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `service_principal_name`
+     - String
+     - The application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.
+  
+   * - `user_name`
+     - String
+     - The email of an active workspace user. Non-admin users can only set this field to their own email.
+  
+  
+## sync
+  
+**`Type: Map`**
+  
+The files and file paths to include or exclude in the bundle. See [_](/dev-tools/bundles/settings.md#sync).
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `exclude`
+     - Sequence
+     - A list of files or folders to exclude from the bundle.
+  
+   * - `include`
+     - Sequence
+     - A list of files or folders to include in the bundle.
+  
+   * - `paths`
+     - Sequence
+     - The local folder paths, which can be outside the bundle root, to synchronize to the workspace when the bundle is deployed.
+  
+  
+## targets
+  
+**`Type: Map`**
+  
+Defines deployment targets for the bundle. See [_](/dev-tools/bundles/settings.md#targets)
+  
+```yaml
+targets:
+  <target-name>:
+    <target-field-name>: <target-field-value>
+```
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `artifacts`
+     - Map
+     - The artifacts to include in the target deployment. See [_](#targets.<name>.artifacts).
+  
+   * - `bundle`
+     - Map
+     - The bundle attributes when deploying to this target. See [_](#targets.<name>.bundle).
+  
+   * - `cluster_id`
+     - String
+     - The ID of the cluster to use for this target.
+  
+   * - `compute_id`
+     - String
+     - Deprecated. The ID of the compute to use for this target.
+  
+   * - `default`
+     - Boolean
+     - Whether this target is the default target.
+  
+   * - `git`
+     - Map
+     - The Git version control settings for the target. See [_](#targets.<name>.git).
+  
+   * - `mode`
+     - String
+     - The deployment mode for the target. Valid values are `development` or `production`. See [_](/dev-tools/bundles/deployment-modes.md).
+  
+   * - `permissions`
+     - Sequence
+     - The permissions for deploying and running the bundle in the target. See [_](#targets.<name>.permissions).
+  
+   * - `presets`
+     - Map
+     - The deployment presets for the target. See [_](#targets.<name>.presets).
+  
+   * - `resources`
+     - Map
+     - The resource definitions for the target. See [_](#targets.<name>.resources).
+  
+   * - `run_as`
+     - Map
+     - The identity to use to run the bundle, see [_](/dev-tools/bundles/run-as.md). See [_](#targets.<name>.run_as).
+  
+   * - `sync`
+     - Map
+     - The local paths to sync to the target workspace when a bundle is run or deployed. See [_](#targets.<name>.sync).
+  
+   * - `variables`
+     - Map
+     - The custom variable definitions for the target. See [_](#targets.<name>.variables).
+  
+   * - `workspace`
+     - Map
+     - The Databricks workspace for the target. See [_](#targets.<name>.workspace).
+  
+  
+### targets.<name>.artifacts
+  
+**`Type: Map`**
+  
+The artifacts to include in the target deployment.
+  
+```yaml
+artifacts:
+  <artifact-name>:
+    <artifact-field-name>: <artifact-field-value>
+```
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `build`
+     - String
+     - An optional set of non-default build commands to run locally before deployment.
+  
+   * - `executable`
+     - String
+     - The executable type. Valid values are `bash`, `sh`, and `cmd`.
+  
+   * - `files`
+     - Sequence
+     - The source files for the artifact. See [_](#targets.<name>.artifacts.<name>.files).
+  
+   * - `path`
+     - String
+     - The location where the built artifact will be saved.
+  
+   * - `type`
+     - String
+     - Required. The type of the artifact. Valid values are `whl`.
+  
+  
+### targets.<name>.artifacts.<name>.files
+  
+**`Type: Sequence`**
+  
+The source files for the artifact.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `source`
+     - String
+     - Required. The path of the files used to build the artifact.
+  
+  
+### targets.<name>.bundle
+  
+**`Type: Map`**
+  
+The bundle attributes when deploying to this target.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `cluster_id`
+     - String
+     - The ID of a cluster to use to run the bundle. See [_](/dev-tools/bundles/settings.md#cluster_id).
+  
+   * - `compute_id`
+     - String
+     - 
+  
+   * - `databricks_cli_version`
+     - String
+     - The Databricks CLI version to use for the bundle. See [_](/dev-tools/bundles/settings.md#databricks_cli_version).
+  
+   * - `deployment`
+     - Map
+     - The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). See [_](#targets.<name>.bundle.deployment).
+  
+   * - `git`
+     - Map
+     - The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). See [_](#targets.<name>.bundle.git).
+  
+   * - `name`
+     - String
+     - The name of the bundle.
+  
+   * - `uuid`
+     - String
+     - Reserved. A Universally Unique Identifier (UUID) for the bundle that uniquely identifies the bundle in internal Databricks systems. This is generated when a bundle project is initialized using a Databricks template (using the `databricks bundle init` command).
+  
+  
+### targets.<name>.bundle.deployment
+  
+**`Type: Map`**
+  
+The definition of the bundle deployment
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `fail_on_active_runs`
+     - Boolean
+     - Whether to fail on active runs. If this is set to true a deployment that is running can be interrupted.
+  
+   * - `lock`
+     - Map
+     - The deployment lock attributes. See [_](#targets.<name>.bundle.deployment.lock).
+  
+  
+### targets.<name>.bundle.deployment.lock
+  
+**`Type: Map`**
+  
+The deployment lock attributes.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `enabled`
+     - Boolean
+     - Whether this lock is enabled.
+  
+   * - `force`
+     - Boolean
+     - Whether to force this lock if it is enabled.
+  
+  
+### targets.<name>.bundle.git
+  
+**`Type: Map`**
+  
+The Git version control details that are associated with your bundle.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `branch`
+     - String
+     - The Git branch name. See [_](/dev-tools/bundles/settings.md#git).
+  
+   * - `origin_url`
+     - String
+     - The origin URL of the repository. See [_](/dev-tools/bundles/settings.md#git).
+  
+  
+### targets.<name>.git
+  
+**`Type: Map`**
+  
+The Git version control settings for the target.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `branch`
+     - String
+     - The Git branch name. See [_](/dev-tools/bundles/settings.md#git).
+  
+   * - `origin_url`
+     - String
+     - The origin URL of the repository. See [_](/dev-tools/bundles/settings.md#git).
+  
+  
+### targets.<name>.permissions
+  
+**`Type: Sequence`**
+  
+The permissions for deploying and running the bundle in the target.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `group_name`
+     - String
+     - The name of the group that has the permission set in level.
+  
+   * - `level`
+     - String
+     - The allowed permission for user, group, service principal defined for this permission.
+  
+   * - `service_principal_name`
+     - String
+     - The name of the service principal that has the permission set in level.
+  
+   * - `user_name`
+     - String
+     - The name of the user that has the permission set in level.
+  
+  
+### targets.<name>.presets
+  
+**`Type: Map`**
+  
+The deployment presets for the target.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `jobs_max_concurrent_runs`
+     - Integer
+     - The maximum concurrent runs for a job.
+  
+   * - `name_prefix`
+     - String
+     - The prefix for job runs of the bundle.
+  
+   * - `pipelines_development`
+     - Boolean
+     - Whether pipeline deployments should be locked in development mode.
+  
+   * - `source_linked_deployment`
+     - Boolean
+     - Whether to link the deployment to the bundle source.
+  
+   * - `tags`
+     - Map
+     - The tags for the bundle deployment.
+  
+   * - `trigger_pause_status`
+     - String
+     - A pause status to apply to all job triggers and schedules. Valid values are PAUSED or UNPAUSED.
+  
+  
+### targets.<name>.resources
+  
+**`Type: Map`**
+  
+The resource definitions for the target.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `apps`
+     - Map
+     - 
+  
+   * - `clusters`
+     - Map
+     - The cluster definitions for the bundle, where each key is the name of a cluster. See [_](/dev-tools/bundles/resources.md#clusters)
+  
+   * - `dashboards`
+     - Map
+     - The dashboard definitions for the bundle, where each key is the name of the dashboard. See [_](/dev-tools/bundles/resources.md#dashboards)
+  
+   * - `experiments`
+     - Map
+     - The experiment definitions for the bundle, where each key is the name of the experiment. See [_](/dev-tools/bundles/resources.md#experiments)
+  
+   * - `jobs`
+     - Map
+     - The job definitions for the bundle, where each key is the name of the job. See [_](/dev-tools/bundles/resources.md#jobs)
+  
+   * - `model_serving_endpoints`
+     - Map
+     - The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint. See [_](/dev-tools/bundles/resources.md#model_serving_endpoints)
+  
+   * - `models`
+     - Map
+     - The model definitions for the bundle, where each key is the name of the model. See [_](/dev-tools/bundles/resources.md#models)
+  
+   * - `pipelines`
+     - Map
+     - The pipeline definitions for the bundle, where each key is the name of the pipeline. See [_](/dev-tools/bundles/resources.md#pipelines)
+  
+   * - `quality_monitors`
+     - Map
+     - The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [_](/dev-tools/bundles/resources.md#quality_monitors)
+  
+   * - `registered_models`
+     - Map
+     - The registered model definitions for the bundle, where each key is the name of the <UC> registered model. See [_](/dev-tools/bundles/resources.md#registered_models)
+  
+   * - `schemas`
+     - Map
+     - The schema definitions for the bundle, where each key is the name of the schema. See [_](/dev-tools/bundles/resources.md#schemas)
+  
+   * - `volumes`
+     - Map
+     - The volume definitions for the bundle, where each key is the name of the volume. See [_](/dev-tools/bundles/resources.md#volumes)
+  
+  
+### targets.<name>.run_as
+  
+**`Type: Map`**
+  
+The identity to use to run the bundle.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `service_principal_name`
+     - String
+     - The application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.
+  
+   * - `user_name`
+     - String
+     - The email of an active workspace user. Non-admin users can only set this field to their own email.
+  
+  
+### targets.<name>.sync
+  
+**`Type: Map`**
+  
+The local paths to sync to the target workspace when a bundle is run or deployed.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `exclude`
+     - Sequence
+     - A list of files or folders to exclude from the bundle.
+  
+   * - `include`
+     - Sequence
+     - A list of files or folders to include in the bundle.
+  
+   * - `paths`
+     - Sequence
+     - The local folder paths, which can be outside the bundle root, to synchronize to the workspace when the bundle is deployed.
+  
+  
+### targets.<name>.variables
+  
+**`Type: Map`**
+  
+The custom variable definitions for the target.
+  
+```yaml
+variables:
+  <variable-name>:
+    <variable-field-name>: <variable-field-value>
+```
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `default`
+     - Any
+     - 
+  
+   * - `description`
+     - String
+     - The description of the variable.
+  
+   * - `lookup`
+     - Map
+     - The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID. See [_](#targets.<name>.variables.<name>.lookup).
+  
+   * - `type`
+     - String
+     - The type of the variable.
+  
+  
+### targets.<name>.variables.<name>.lookup
+  
+**`Type: Map`**
+  
+The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `alert`
+     - String
+     - 
+  
+   * - `cluster`
+     - String
+     - 
+  
+   * - `cluster_policy`
+     - String
+     - 
+  
+   * - `dashboard`
+     - String
+     - 
+  
+   * - `instance_pool`
+     - String
+     - 
+  
+   * - `job`
+     - String
+     - 
+  
+   * - `metastore`
+     - String
+     - 
+  
+   * - `notification_destination`
+     - String
+     - 
+  
+   * - `pipeline`
+     - String
+     - 
+  
+   * - `query`
+     - String
+     - 
+  
+   * - `service_principal`
+     - String
+     - 
+  
+   * - `warehouse`
+     - String
+     - 
+  
+  
+### targets.<name>.workspace
+  
+**`Type: Map`**
+  
+The Databricks workspace for the target.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `artifact_path`
+     - String
+     - The artifact path to use within the workspace for both deployments and workflow runs
+  
+   * - `auth_type`
+     - String
+     - The authentication type.
+  
+   * - `azure_client_id`
+     - String
+     - The Azure client ID
+  
+   * - `azure_environment`
+     - String
+     - The Azure environment
+  
+   * - `azure_login_app_id`
+     - String
+     - The Azure login app ID
+  
+   * - `azure_tenant_id`
+     - String
+     - The Azure tenant ID
+  
+   * - `azure_use_msi`
+     - Boolean
+     - Whether to use MSI for Azure
+  
+   * - `azure_workspace_resource_id`
+     - String
+     - The Azure workspace resource ID
+  
+   * - `client_id`
+     - String
+     - The client ID for the workspace
+  
+   * - `file_path`
+     - String
+     - The file path to use within the workspace for both deployments and workflow runs
+  
+   * - `google_service_account`
+     - String
+     - The Google service account name
+  
+   * - `host`
+     - String
+     - The Databricks workspace host URL
+  
+   * - `profile`
+     - String
+     - The Databricks workspace profile name
+  
+   * - `resource_path`
+     - String
+     - The workspace resource path
+  
+   * - `root_path`
+     - String
+     - The Databricks workspace root path
+  
+   * - `state_path`
+     - String
+     - The workspace state path
+  
+  
+## variables
+  
+**`Type: Map`**
+  
+Defines a custom variable for the bundle. See [_](/dev-tools/bundles/settings.md#variables).
+  
+```yaml
+variables:
+  <variable-name>:
+    <variable-field-name>: <variable-field-value>
+```
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `default`
+     - Any
+     - 
+  
+   * - `description`
+     - String
+     - The description of the variable
+  
+   * - `lookup`
+     - Map
+     - The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID. See [_](#variables.<name>.lookup).
+  
+   * - `type`
+     - String
+     - The type of the variable.
+  
+  
+### variables.<name>.lookup
+  
+**`Type: Map`**
+  
+The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `alert`
+     - String
+     - 
+  
+   * - `cluster`
+     - String
+     - 
+  
+   * - `cluster_policy`
+     - String
+     - 
+  
+   * - `dashboard`
+     - String
+     - 
+  
+   * - `instance_pool`
+     - String
+     - 
+  
+   * - `job`
+     - String
+     - 
+  
+   * - `metastore`
+     - String
+     - 
+  
+   * - `notification_destination`
+     - String
+     - 
+  
+   * - `pipeline`
+     - String
+     - 
+  
+   * - `query`
+     - String
+     - 
+  
+   * - `service_principal`
+     - String
+     - 
+  
+   * - `warehouse`
+     - String
+     - 
+  
+  
+## workspace
+  
+**`Type: Map`**
+  
+Defines the Databricks workspace for the bundle. See [_](/dev-tools/bundles/settings.md#workspace).
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `artifact_path`
+     - String
+     - The artifact path to use within the workspace for both deployments and workflow runs
+  
+   * - `auth_type`
+     - String
+     - The authentication type.
+  
+   * - `azure_client_id`
+     - String
+     - The Azure client ID
+  
+   * - `azure_environment`
+     - String
+     - The Azure environment
+  
+   * - `azure_login_app_id`
+     - String
+     - The Azure login app ID
+  
+   * - `azure_tenant_id`
+     - String
+     - The Azure tenant ID
+  
+   * - `azure_use_msi`
+     - Boolean
+     - Whether to use MSI for Azure
+  
+   * - `azure_workspace_resource_id`
+     - String
+     - The Azure workspace resource ID
+  
+   * - `client_id`
+     - String
+     - The client ID for the workspace
+  
+   * - `file_path`
+     - String
+     - The file path to use within the workspace for both deployments and workflow runs
+  
+   * - `google_service_account`
+     - String
+     - The Google service account name
+  
+   * - `host`
+     - String
+     - The Databricks workspace host URL
+  
+   * - `profile`
+     - String
+     - The Databricks workspace profile name
+  
+   * - `resource_path`
+     - String
+     - The workspace resource path
+  
+   * - `root_path`
+     - String
+     - The Databricks workspace root path
+  
+   * - `state_path`
+     - String
+     - The workspace state path
+  
\ No newline at end of file
diff --git a/bundle/docsgen/output/resources.md b/bundle/docsgen/output/resources.md
new file mode 100644
index 000000000..ff80ee635
--- /dev/null
+++ b/bundle/docsgen/output/resources.md
@@ -0,0 +1,8189 @@
+<!-- DO NOT EDIT. This file is autogenerated with https://github.com/databricks/cli -->
+---
+description: Learn about resources supported by Databricks Asset Bundles and how to configure them.
+---
+
+# <DABS> resources
+
+<DABS> allows you to specify information about the <Databricks> resources used by the bundle in the `resources` mapping in the bundle configuration. See [resources mapping](/dev-tools/bundles/settings.md#resources) and [resources key reference](/dev-tools/bundles/reference.md#resources).
+
+This article outlines supported resource types for bundles and provides details and an example for each supported type. For additional examples, see [_](/dev-tools/bundles/resource-examples.md).
+
+## <a id="resource-types"></a> Supported resources
+
+The following table lists supported resource types for bundles. Some resources can be created by defining them in a bundle and deploying the bundle, and some resources only support referencing an existing resource to include in the bundle.
+
+Resources are defined using the corresponding [Databricks REST API](/api/workspace/introduction) object's create operation request payload, where the object's supported fields, expressed as YAML, are the resource's supported properties. Links to documentation for each resource's corresponding payloads are listed in the table.
+
+.. tip:: The `databricks bundle validate` command returns warnings if unknown resource properties are found in bundle configuration files.
+
+
+.. list-table::
+    :header-rows: 1
+
+    * - Resource
+      - Create support
+      - Corresponding REST API object
+
+    * - [cluster](#cluster)
+      - ✓
+      - [Cluster object](/api/workspace/clusters/create)
+
+    * - [dashboard](#dashboard)
+      -
+      - [Dashboard object](/api/workspace/lakeview/create)
+
+    * - [experiment](#experiment)
+      - ✓
+      - [Experiment object](/api/workspace/experiments/createexperiment)
+
+    * - [job](#job)
+      - ✓
+      - [Job object](/api/workspace/jobs/create)
+
+    * - [model (legacy)](#model-legacy)
+      - ✓
+      - [Model (legacy) object](/api/workspace/modelregistry/createmodel)
+
+    * - [model_serving_endpoint](#model-serving-endpoint)
+      - ✓
+      - [Model serving endpoint object](/api/workspace/servingendpoints/create)
+
+    * - [pipeline](#pipeline)
+      - ✓
+      - [Pipeline object](/api/workspace/pipelines/create)
+
+    * - [quality_monitor](#quality-monitor)
+      - ✓
+      - [Quality monitor object](/api/workspace/qualitymonitors/create)
+
+    * - [registered_model](#registered-model) (<UC>)
+      - ✓
+      - [Registered model object](/api/workspace/registeredmodels/create)
+
+    * - [schema](#schema) (<UC>)
+      - ✓
+      - [Schema object](/api/workspace/schemas/create)
+
+    * - [volume](#volume) (<UC>)
+      - ✓
+      - [Volume object](/api/workspace/volumes/create)
+
+  
+## apps
+  
+**`Type: Map`**
+  
+
+  
+```yaml
+apps:
+  <app-name>:
+    <app-field-name>: <app-field-value>
+```
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `active_deployment`
+     - Map
+     - See [_](#apps.<name>.active_deployment).
+  
+   * - `app_status`
+     - Map
+     - See [_](#apps.<name>.app_status).
+  
+   * - `compute_status`
+     - Map
+     - See [_](#apps.<name>.compute_status).
+  
+   * - `config`
+     - Map
+     - 
+  
+   * - `create_time`
+     - String
+     - 
+  
+   * - `creator`
+     - String
+     - 
+  
+   * - `default_source_code_path`
+     - String
+     - 
+  
+   * - `description`
+     - String
+     - 
+  
+   * - `name`
+     - String
+     - 
+  
+   * - `pending_deployment`
+     - Map
+     - See [_](#apps.<name>.pending_deployment).
+  
+   * - `permissions`
+     - Sequence
+     - See [_](#apps.<name>.permissions).
+  
+   * - `resources`
+     - Sequence
+     - See [_](#apps.<name>.resources).
+  
+   * - `service_principal_client_id`
+     - String
+     - 
+  
+   * - `service_principal_id`
+     - Integer
+     - 
+  
+   * - `service_principal_name`
+     - String
+     - 
+  
+   * - `source_code_path`
+     - String
+     - 
+  
+   * - `update_time`
+     - String
+     - 
+  
+   * - `updater`
+     - String
+     - 
+  
+   * - `url`
+     - String
+     - 
+  
+  
+### apps.<name>.active_deployment
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `create_time`
+     - String
+     - 
+  
+   * - `creator`
+     - String
+     - 
+  
+   * - `deployment_artifacts`
+     - Map
+     - See [_](#apps.<name>.active_deployment.deployment_artifacts).
+  
+   * - `deployment_id`
+     - String
+     - 
+  
+   * - `mode`
+     - String
+     - 
+  
+   * - `source_code_path`
+     - String
+     - 
+  
+   * - `status`
+     - Map
+     - See [_](#apps.<name>.active_deployment.status).
+  
+   * - `update_time`
+     - String
+     - 
+  
+  
+### apps.<name>.active_deployment.deployment_artifacts
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `source_code_path`
+     - String
+     - 
+  
+  
+### apps.<name>.active_deployment.status
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `message`
+     - String
+     - 
+  
+   * - `state`
+     - String
+     - 
+  
+  
+### apps.<name>.app_status
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `message`
+     - String
+     - 
+  
+   * - `state`
+     - String
+     - 
+  
+  
+### apps.<name>.compute_status
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `message`
+     - String
+     - 
+  
+   * - `state`
+     - String
+     - State of the app compute.
+  
+  
+### apps.<name>.pending_deployment
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `create_time`
+     - String
+     - 
+  
+   * - `creator`
+     - String
+     - 
+  
+   * - `deployment_artifacts`
+     - Map
+     - See [_](#apps.<name>.pending_deployment.deployment_artifacts).
+  
+   * - `deployment_id`
+     - String
+     - 
+  
+   * - `mode`
+     - String
+     - 
+  
+   * - `source_code_path`
+     - String
+     - 
+  
+   * - `status`
+     - Map
+     - See [_](#apps.<name>.pending_deployment.status).
+  
+   * - `update_time`
+     - String
+     - 
+  
+  
+### apps.<name>.pending_deployment.deployment_artifacts
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `source_code_path`
+     - String
+     - 
+  
+  
+### apps.<name>.pending_deployment.status
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `message`
+     - String
+     - 
+  
+   * - `state`
+     - String
+     - 
+  
+  
+### apps.<name>.permissions
+  
+**`Type: Sequence`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `group_name`
+     - String
+     - The name of the group that has the permission set in level.
+  
+   * - `level`
+     - String
+     - The allowed permission for user, group, service principal defined for this permission.
+  
+   * - `service_principal_name`
+     - String
+     - The name of the service principal that has the permission set in level.
+  
+   * - `user_name`
+     - String
+     - The name of the user that has the permission set in level.
+  
+  
+### apps.<name>.resources
+  
+**`Type: Sequence`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `description`
+     - String
+     - Description of the App Resource.
+  
+   * - `job`
+     - Map
+     - See [_](#apps.<name>.resources.job).
+  
+   * - `name`
+     - String
+     - Name of the App Resource.
+  
+   * - `secret`
+     - Map
+     - See [_](#apps.<name>.resources.secret).
+  
+   * - `serving_endpoint`
+     - Map
+     - See [_](#apps.<name>.resources.serving_endpoint).
+  
+   * - `sql_warehouse`
+     - Map
+     - See [_](#apps.<name>.resources.sql_warehouse).
+  
+  
+### apps.<name>.resources.job
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `id`
+     - String
+     - 
+  
+   * - `permission`
+     - String
+     - 
+  
+  
+### apps.<name>.resources.secret
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `key`
+     - String
+     - 
+  
+   * - `permission`
+     - String
+     - Permission to grant on the secret scope. Supported permissions are: "READ", "WRITE", "MANAGE".
+  
+   * - `scope`
+     - String
+     - 
+  
+  
+### apps.<name>.resources.serving_endpoint
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `name`
+     - String
+     - 
+  
+   * - `permission`
+     - String
+     - 
+  
+  
+### apps.<name>.resources.sql_warehouse
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `id`
+     - String
+     - 
+  
+   * - `permission`
+     - String
+     - 
+  
+  
+## clusters
+  
+**`Type: Map`**
+  
+The cluster resource defines an [all-purpose cluster](/api/workspace/clusters/create).
+  
+```yaml
+clusters:
+  <cluster-name>:
+    <cluster-field-name>: <cluster-field-value>
+```
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `apply_policy_default_values`
+     - Boolean
+     - When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied.
+  
+   * - `autoscale`
+     - Map
+     - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#clusters.<name>.autoscale).
+  
+   * - `autotermination_minutes`
+     - Integer
+     - Automatically terminates the cluster after it is inactive for this time in minutes. If not set, this cluster will not be automatically terminated. If specified, the threshold must be between 10 and 10000 minutes. Users can also set this value to 0 to explicitly disable automatic termination.
+  
+   * - `aws_attributes`
+     - Map
+     - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#clusters.<name>.aws_attributes).
+  
+   * - `azure_attributes`
+     - Map
+     - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#clusters.<name>.azure_attributes).
+  
+   * - `cluster_log_conf`
+     - Map
+     - The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. See [_](#clusters.<name>.cluster_log_conf).
+  
+   * - `cluster_name`
+     - String
+     - Cluster name requested by the user. This doesn't have to be unique. If not specified at creation, the cluster name will be an empty string. 
+  
+   * - `custom_tags`
+     - Map
+     - Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS instances and EBS volumes) with these tags in addition to `default_tags`. Notes:  - Currently, Databricks allows at most 45 custom tags  - Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags
+  
+   * - `data_security_mode`
+     - String
+     - Data security mode decides what data governance model to use when accessing data from a cluster.  The following modes can only be used with `kind`. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`.  The following modes can be used regardless of `kind`. * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.  The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions:  * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. 
+  
+   * - `docker_image`
+     - Map
+     - See [_](#clusters.<name>.docker_image).
+  
+   * - `driver_instance_pool_id`
+     - String
+     - The optional ID of the instance pool for the driver of the cluster belongs. The pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not assigned.
+  
+   * - `driver_node_type_id`
+     - String
+     - The node type of the Spark driver. Note that this field is optional; if unset, the driver node type will be set as the same value as `node_type_id` defined above. 
+  
+   * - `enable_elastic_disk`
+     - Boolean
+     - Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk space when its Spark workers are running low on disk space. This feature requires specific AWS permissions to function correctly - refer to the User Guide for more details.
+  
+   * - `enable_local_disk_encryption`
+     - Boolean
+     - Whether to enable LUKS on cluster VMs' local disks
+  
+   * - `gcp_attributes`
+     - Map
+     - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#clusters.<name>.gcp_attributes).
+  
+   * - `init_scripts`
+     - Sequence
+     - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `<destination>/<cluster-ID>/init_scripts`. See [_](#clusters.<name>.init_scripts).
+  
+   * - `instance_pool_id`
+     - String
+     - The optional ID of the instance pool to which the cluster belongs.
+  
+   * - `is_single_node`
+     - Boolean
+     - This field can only be used with `kind`.  When set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, and `num_workers` 
+  
+   * - `kind`
+     - String
+     - 
+  
+   * - `node_type_id`
+     - String
+     - This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or compute intensive workloads. A list of available node types can be retrieved by using the :method:clusters/listNodeTypes API call. 
+  
+   * - `num_workers`
+     - Integer
+     - Number of worker nodes that this cluster should have. A cluster has one Spark Driver and `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.  Note: When reading the properties of a cluster, this field reflects the desired number of workers rather than the actual current number of workers. For instance, if a cluster is resized from 5 to 10 workers, this field will immediately be updated to reflect the target size of 10 workers, whereas the workers listed in `spark_info` will gradually increase from 5 to 10 as the new nodes are provisioned.
+  
+   * - `permissions`
+     - Sequence
+     - See [_](#clusters.<name>.permissions).
+  
+   * - `policy_id`
+     - String
+     - The ID of the cluster policy used to create the cluster if applicable.
+  
+   * - `runtime_engine`
+     - String
+     - Determines the cluster's runtime engine, either standard or Photon.  This field is not compatible with legacy `spark_version` values that contain `-photon-`. Remove `-photon-` from the `spark_version` and set `runtime_engine` to `PHOTON`.  If left unspecified, the runtime engine defaults to standard unless the spark_version contains -photon-, in which case Photon will be used. 
+  
+   * - `single_user_name`
+     - String
+     - Single user name if data_security_mode is `SINGLE_USER`
+  
+   * - `spark_conf`
+     - Map
+     - An object containing a set of optional, user-specified Spark configuration key-value pairs. Users can also pass in a string of extra JVM options to the driver and the executors via `spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively. 
+  
+   * - `spark_env_vars`
+     - Map
+     - An object containing a set of optional, user-specified environment variable key-value pairs. Please note that key-value pair of the form (X,Y) will be exported as is (i.e., `export X='Y'`) while launching the driver and workers.  In order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending them to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all default databricks managed environmental variables are included as well.  Example Spark environment variables: `{"SPARK_WORKER_MEMORY": "28000m", "SPARK_LOCAL_DIRS": "/local_disk0"}` or `{"SPARK_DAEMON_JAVA_OPTS": "$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true"}`
+  
+   * - `spark_version`
+     - String
+     - The Spark version of the cluster, e.g. `3.3.x-scala2.11`. A list of available Spark versions can be retrieved by using the :method:clusters/sparkVersions API call. 
+  
+   * - `ssh_public_keys`
+     - Sequence
+     - SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified.
+  
+   * - `use_ml_runtime`
+     - Boolean
+     - This field can only be used with `kind`.  `effective_spark_version` is determined by `spark_version` (DBR release), this field `use_ml_runtime`, and whether `node_type_id` is gpu node or not. 
+  
+   * - `workload_type`
+     - Map
+     - See [_](#clusters.<name>.workload_type).
+  
+  
+**Example**
+  
+The following example creates a cluster named `my_cluster` and sets that as the cluster to use to run the notebook in `my_job`:
+
+```yaml
+bundle:
+  name: clusters
+
+resources:
+  clusters:
+    my_cluster:
+      num_workers: 2
+      node_type_id: "i3.xlarge"
+      autoscale:
+        min_workers: 2
+        max_workers: 7
+      spark_version: "13.3.x-scala2.12"
+      spark_conf:
+        "spark.executor.memory": "2g"
+
+  jobs:
+    my_job:
+      tasks:
+        - task_key: test_task
+          notebook_task:
+            notebook_path: "./src/my_notebook.py"
+```
+  
+### clusters.<name>.autoscale
+  
+**`Type: Map`**
+  
+Parameters needed in order to automatically scale clusters up and down based on load.
+Note: autoscaling works best with DB runtime versions 3.0 or later.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `max_workers`
+     - Integer
+     - The maximum number of workers to which the cluster can scale up when overloaded. Note that `max_workers` must be strictly greater than `min_workers`.
+  
+   * - `min_workers`
+     - Integer
+     - The minimum number of workers to which the cluster can scale down when underutilized. It is also the initial number of workers the cluster will have after creation.
+  
+  
+### clusters.<name>.aws_attributes
+  
+**`Type: Map`**
+  
+Attributes related to clusters running on Amazon Web Services.
+If not specified at cluster creation, a set of default values will be used.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `availability`
+     - String
+     - Availability type used for all subsequent nodes past the `first_on_demand` ones.  Note: If `first_on_demand` is zero, this availability type will be used for the entire cluster. 
+  
+   * - `ebs_volume_count`
+     - Integer
+     - The number of volumes launched for each instance. Users can choose up to 10 volumes. This feature is only enabled for supported node types. Legacy node types cannot specify custom EBS volumes. For node types with no instance store, at least one EBS volume needs to be specified; otherwise, cluster creation will fail.  These EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc. Instance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc.  If EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for scratch storage because heterogenously sized scratch devices can lead to inefficient disk utilization. If no EBS volumes are attached, Databricks will configure Spark to use instance store volumes.  Please note that if EBS volumes are specified, then the Spark configuration `spark.local.dir` will be overridden.
+  
+   * - `ebs_volume_iops`
+     - Integer
+     - If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used.
+  
+   * - `ebs_volume_size`
+     - Integer
+     - The size of each EBS volume (in GiB) launched for each instance. For general purpose SSD, this value must be within the range 100 - 4096. For throughput optimized HDD, this value must be within the range 500 - 4096.
+  
+   * - `ebs_volume_throughput`
+     - Integer
+     - If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used.
+  
+   * - `ebs_volume_type`
+     - String
+     - The type of EBS volumes that will be launched with this cluster.
+  
+   * - `first_on_demand`
+     - Integer
+     - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. If this value is greater than 0, the cluster driver node in particular will be placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster.
+  
+   * - `instance_profile_arn`
+     - String
+     - Nodes for this cluster will only be placed on AWS instances with this instance profile. If ommitted, nodes will be placed on instances without an IAM instance profile. The instance profile must have previously been added to the Databricks environment by an account administrator.  This feature may only be available to certain customer plans.  If this field is ommitted, we will pull in the default from the conf if it exists.
+  
+   * - `spot_bid_price_percent`
+     - Integer
+     - The bid price for AWS spot instances, as a percentage of the corresponding instance type's on-demand price. For example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot instance, then the bid price is half of the price of on-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice the price of on-demand `r3.xlarge` instances. If not specified, the default value is 100. When spot instances are requested for this cluster, only spot instances whose bid price percentage matches this field will be considered. Note that, for safety, we enforce this field to be no more than 10000.  The default value and documentation here should be kept consistent with CommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent.
+  
+   * - `zone_id`
+     - String
+     - Identifier for the availability zone/datacenter in which the cluster resides. This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. This is an optional field at cluster creation, and if not specified, a default zone will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. The list of available zones as well as the default value can be found by using the `List Zones` method.
+  
+  
+### clusters.<name>.azure_attributes
+  
+**`Type: Map`**
+  
+Attributes related to clusters running on Microsoft Azure.
+If not specified at cluster creation, a set of default values will be used.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `availability`
+     - String
+     - Availability type used for all subsequent nodes past the `first_on_demand` ones. Note: If `first_on_demand` is zero (which only happens on pool clusters), this availability type will be used for the entire cluster.
+  
+   * - `first_on_demand`
+     - Integer
+     - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster.
+  
+   * - `log_analytics_info`
+     - Map
+     - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#clusters.<name>.azure_attributes.log_analytics_info).
+  
+   * - `spot_bid_max_price`
+     - Any
+     - The max bid price to be used for Azure spot instances. The Max price for the bid cannot be higher than the on-demand price of the instance. If not specified, the default value is -1, which specifies that the instance cannot be evicted on the basis of price, and only on the basis of availability. Further, the value should > 0 or -1.
+  
+  
+### clusters.<name>.azure_attributes.log_analytics_info
+  
+**`Type: Map`**
+  
+Defines values necessary to configure and run Azure Log Analytics agent
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `log_analytics_primary_key`
+     - String
+     - <needs content added>
+  
+   * - `log_analytics_workspace_id`
+     - String
+     - <needs content added>
+  
+  
+### clusters.<name>.cluster_log_conf
+  
+**`Type: Map`**
+  
+The configuration for delivering spark logs to a long-term storage destination.
+Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified
+for one cluster. If the conf is given, the logs will be delivered to the destination every
+`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while
+the destination of executor logs is `$destination/$clusterId/executor`.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `dbfs`
+     - Map
+     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#clusters.<name>.cluster_log_conf.dbfs).
+  
+   * - `s3`
+     - Map
+     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#clusters.<name>.cluster_log_conf.s3).
+  
+  
+### clusters.<name>.cluster_log_conf.dbfs
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - dbfs destination, e.g. `dbfs:/my/path`
+  
+  
+### clusters.<name>.cluster_log_conf.s3
+  
+**`Type: Map`**
+  
+destination and either the region or endpoint need to be provided. e.g.
+`{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }`
+Cluster iam role is used to access s3, please make sure the cluster iam role in
+`instance_profile_arn` has permission to write data to the s3 destination.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `canned_acl`
+     - String
+     - (Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`. If `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on the destination bucket and prefix. The full list of possible canned acl can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl. Please also note that by default only the object owner gets full controls. If you are using cross account role for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to read the logs.
+  
+   * - `destination`
+     - String
+     - S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using cluster iam role, please make sure you set cluster iam role and the role has write access to the destination. Please also note that you cannot use AWS keys to deliver logs.
+  
+   * - `enable_encryption`
+     - Boolean
+     - (Optional) Flag to enable server side encryption, `false` by default.
+  
+   * - `encryption_type`
+     - String
+     - (Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when encryption is enabled and the default type is `sse-s3`.
+  
+   * - `endpoint`
+     - String
+     - S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
+  
+   * - `kms_key`
+     - String
+     - (Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`.
+  
+   * - `region`
+     - String
+     - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
+  
+  
+### clusters.<name>.docker_image
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `basic_auth`
+     - Map
+     - See [_](#clusters.<name>.docker_image.basic_auth).
+  
+   * - `url`
+     - String
+     - URL of the docker image.
+  
+  
+### clusters.<name>.docker_image.basic_auth
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `password`
+     - String
+     - Password of the user
+  
+   * - `username`
+     - String
+     - Name of the user
+  
+  
+### clusters.<name>.gcp_attributes
+  
+**`Type: Map`**
+  
+Attributes related to clusters running on Google Cloud Platform.
+If not specified at cluster creation, a set of default values will be used.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `availability`
+     - String
+     - This field determines whether the instance pool will contain preemptible VMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable.
+  
+   * - `boot_disk_size`
+     - Integer
+     - boot disk size in GB
+  
+   * - `google_service_account`
+     - String
+     - If provided, the cluster will impersonate the google service account when accessing gcloud services (like GCS). The google service account must have previously been added to the Databricks environment by an account administrator.
+  
+   * - `local_ssd_count`
+     - Integer
+     - If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type.
+  
+   * - `use_preemptible_executors`
+     - Boolean
+     - This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default). Note: Soon to be deprecated, use the availability field instead.
+  
+   * - `zone_id`
+     - String
+     - Identifier for the availability zone in which the cluster resides. This can be one of the following: - "HA" => High availability, spread nodes across availability zones for a Databricks deployment region [default] - "AUTO" => Databricks picks an availability zone to schedule the cluster on. - A GCP availability zone => Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones.
+  
+  
+### clusters.<name>.init_scripts
+  
+**`Type: Sequence`**
+  
+The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `<destination>/<cluster-ID>/init_scripts`.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `abfss`
+     - Map
+     - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>" } }. See [_](#clusters.<name>.init_scripts.abfss).
+  
+   * - `dbfs`
+     - Map
+     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#clusters.<name>.init_scripts.dbfs).
+  
+   * - `file`
+     - Map
+     - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#clusters.<name>.init_scripts.file).
+  
+   * - `gcs`
+     - Map
+     - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#clusters.<name>.init_scripts.gcs).
+  
+   * - `s3`
+     - Map
+     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#clusters.<name>.init_scripts.s3).
+  
+   * - `volumes`
+     - Map
+     - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#clusters.<name>.init_scripts.volumes).
+  
+   * - `workspace`
+     - Map
+     - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#clusters.<name>.init_scripts.workspace).
+  
+  
+### clusters.<name>.init_scripts.abfss
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "abfss" : { "destination" : "abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>" } }
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - abfss destination, e.g. `abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>`.
+  
+  
+### clusters.<name>.init_scripts.dbfs
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - dbfs destination, e.g. `dbfs:/my/path`
+  
+  
+### clusters.<name>.init_scripts.file
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "file" : { "destination" : "file:/my/local/file.sh" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - local file destination, e.g. `file:/my/local/file.sh`
+  
+  
+### clusters.<name>.init_scripts.gcs
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - GCS destination/URI, e.g. `gs://my-bucket/some-prefix`
+  
+  
+### clusters.<name>.init_scripts.s3
+  
+**`Type: Map`**
+  
+destination and either the region or endpoint need to be provided. e.g.
+`{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }`
+Cluster iam role is used to access s3, please make sure the cluster iam role in
+`instance_profile_arn` has permission to write data to the s3 destination.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `canned_acl`
+     - String
+     - (Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`. If `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on the destination bucket and prefix. The full list of possible canned acl can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl. Please also note that by default only the object owner gets full controls. If you are using cross account role for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to read the logs.
+  
+   * - `destination`
+     - String
+     - S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using cluster iam role, please make sure you set cluster iam role and the role has write access to the destination. Please also note that you cannot use AWS keys to deliver logs.
+  
+   * - `enable_encryption`
+     - Boolean
+     - (Optional) Flag to enable server side encryption, `false` by default.
+  
+   * - `encryption_type`
+     - String
+     - (Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when encryption is enabled and the default type is `sse-s3`.
+  
+   * - `endpoint`
+     - String
+     - S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
+  
+   * - `kms_key`
+     - String
+     - (Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`.
+  
+   * - `region`
+     - String
+     - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
+  
+  
+### clusters.<name>.init_scripts.volumes
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`
+  
+  
+### clusters.<name>.init_scripts.workspace
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`
+  
+  
+### clusters.<name>.permissions
+  
+**`Type: Sequence`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `group_name`
+     - String
+     - The name of the group that has the permission set in level.
+  
+   * - `level`
+     - String
+     - The allowed permission for user, group, service principal defined for this permission.
+  
+   * - `service_principal_name`
+     - String
+     - The name of the service principal that has the permission set in level.
+  
+   * - `user_name`
+     - String
+     - The name of the user that has the permission set in level.
+  
+  
+### clusters.<name>.workload_type
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `clients`
+     - Map
+     -  defined what type of clients can use the cluster. E.g. Notebooks, Jobs. See [_](#clusters.<name>.workload_type.clients).
+  
+  
+### clusters.<name>.workload_type.clients
+  
+**`Type: Map`**
+  
+ defined what type of clients can use the cluster. E.g. Notebooks, Jobs
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `jobs`
+     - Boolean
+     - With jobs set, the cluster can be used for jobs
+  
+   * - `notebooks`
+     - Boolean
+     - With notebooks set, this cluster can be used for notebooks
+  
+  
+## dashboards
+  
+**`Type: Map`**
+  
+The dashboard resource allows you to manage [AI/BI dashboards](/api/workspace/lakeview/create) in a bundle. For information about AI/BI dashboards, see [_](/dashboards/index.md).
+  
+```yaml
+dashboards:
+  <dashboard-name>:
+    <dashboard-field-name>: <dashboard-field-value>
+```
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `create_time`
+     - String
+     - The timestamp of when the dashboard was created.
+  
+   * - `dashboard_id`
+     - String
+     - UUID identifying the dashboard.
+  
+   * - `display_name`
+     - String
+     - The display name of the dashboard.
+  
+   * - `embed_credentials`
+     - Boolean
+     - 
+  
+   * - `etag`
+     - String
+     - The etag for the dashboard. Can be optionally provided on updates to ensure that the dashboard has not been modified since the last read. This field is excluded in List Dashboards responses.
+  
+   * - `file_path`
+     - String
+     - 
+  
+   * - `lifecycle_state`
+     - String
+     - The state of the dashboard resource. Used for tracking trashed status.
+  
+   * - `parent_path`
+     - String
+     - The workspace path of the folder containing the dashboard. Includes leading slash and no trailing slash. This field is excluded in List Dashboards responses.
+  
+   * - `path`
+     - String
+     - The workspace path of the dashboard asset, including the file name. Exported dashboards always have the file extension `.lvdash.json`. This field is excluded in List Dashboards responses.
+  
+   * - `permissions`
+     - Sequence
+     - See [_](#dashboards.<name>.permissions).
+  
+   * - `serialized_dashboard`
+     - Any
+     - The contents of the dashboard in serialized string form. This field is excluded in List Dashboards responses. Use the [get dashboard API](https://docs.databricks.com/api/workspace/lakeview/get) to retrieve an example response, which includes the `serialized_dashboard` field. This field provides the structure of the JSON string that represents the dashboard's layout and components.
+  
+   * - `update_time`
+     - String
+     - The timestamp of when the dashboard was last updated by the user. This field is excluded in List Dashboards responses.
+  
+   * - `warehouse_id`
+     - String
+     - The warehouse ID used to run the dashboard.
+  
+  
+**Example**
+  
+The following example includes and deploys the sample __NYC Taxi Trip Analysis__ dashboard to the Databricks workspace.
+
+``` yaml
+resources:
+  dashboards:
+    nyc_taxi_trip_analysis:
+      display_name: "NYC Taxi Trip Analysis"
+      file_path: ../src/nyc_taxi_trip_analysis.lvdash.json
+      warehouse_id: ${var.warehouse_id}
+```
+If you use the UI to modify the dashboard, modifications made through the UI are not applied to the dashboard JSON file in the local bundle unless you explicitly update it using `bundle generate`. You can use the `--watch` option to continuously poll and retrieve changes to the dashboard. See [_](/dev-tools/cli/bundle-commands.md#generate).
+
+In addition, if you attempt to deploy a bundle that contains a dashboard JSON file that is different than the one in the remote workspace, an error will occur. To force the deploy and overwrite the dashboard in the remote workspace with the local one, use the `--force` option. See [_](/dev-tools/cli/bundle-commands.md#deploy).
+  
+### dashboards.<name>.permissions
+  
+**`Type: Sequence`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `group_name`
+     - String
+     - The name of the group that has the permission set in level.
+  
+   * - `level`
+     - String
+     - The allowed permission for user, group, service principal defined for this permission.
+  
+   * - `service_principal_name`
+     - String
+     - The name of the service principal that has the permission set in level.
+  
+   * - `user_name`
+     - String
+     - The name of the user that has the permission set in level.
+  
+  
+## experiments
+  
+**`Type: Map`**
+  
+The experiment resource allows you to define [MLflow experiments](/api/workspace/experiments/createexperiment) in a bundle. For information about MLflow experiments, see [_](/mlflow/experiments.md).
+  
+```yaml
+experiments:
+  <experiment-name>:
+    <experiment-field-name>: <experiment-field-value>
+```
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `artifact_location`
+     - String
+     - Location where artifacts for the experiment are stored.
+  
+   * - `creation_time`
+     - Integer
+     - Creation time
+  
+   * - `experiment_id`
+     - String
+     - Unique identifier for the experiment.
+  
+   * - `last_update_time`
+     - Integer
+     - Last update time
+  
+   * - `lifecycle_stage`
+     - String
+     - Current life cycle stage of the experiment: "active" or "deleted". Deleted experiments are not returned by APIs.
+  
+   * - `name`
+     - String
+     - Human readable name that identifies the experiment.
+  
+   * - `permissions`
+     - Sequence
+     - See [_](#experiments.<name>.permissions).
+  
+   * - `tags`
+     - Sequence
+     - Tags: Additional metadata key-value pairs. See [_](#experiments.<name>.tags).
+  
+  
+**Example**
+  
+The following example defines an experiment that all users can view:
+
+```yaml
+resources:
+  experiments:
+    experiment:
+      name: my_ml_experiment
+      permissions:
+        - level: CAN_READ
+          group_name: users
+      description: MLflow experiment used to track runs
+```
+  
+### experiments.<name>.permissions
+  
+**`Type: Sequence`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `group_name`
+     - String
+     - The name of the group that has the permission set in level.
+  
+   * - `level`
+     - String
+     - The allowed permission for user, group, service principal defined for this permission.
+  
+   * - `service_principal_name`
+     - String
+     - The name of the service principal that has the permission set in level.
+  
+   * - `user_name`
+     - String
+     - The name of the user that has the permission set in level.
+  
+  
+### experiments.<name>.tags
+  
+**`Type: Sequence`**
+  
+Tags: Additional metadata key-value pairs.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `key`
+     - String
+     - The tag key.
+  
+   * - `value`
+     - String
+     - The tag value.
+  
+  
+## jobs
+  
+**`Type: Map`**
+  
+The job resource allows you to define [jobs and their corresponding tasks](/api/workspace/jobs/create) in your bundle. For information about jobs, see [_](/jobs/index.md). For a tutorial that uses a <DABS> template to create a job, see [_](/dev-tools/bundles/jobs-tutorial.md).
+  
+```yaml
+jobs:
+  <job-name>:
+    <job-field-name>: <job-field-value>
+```
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `budget_policy_id`
+     - String
+     - The id of the user specified budget policy to use for this job. If not specified, a default budget policy may be applied when creating or modifying the job. See `effective_budget_policy_id` for the budget policy used by this workload.
+  
+   * - `continuous`
+     - Map
+     - An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used. See [_](#jobs.<name>.continuous).
+  
+   * - `deployment`
+     - Map
+     - Deployment information for jobs managed by external sources. See [_](#jobs.<name>.deployment).
+  
+   * - `description`
+     - String
+     - An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding.
+  
+   * - `edit_mode`
+     - String
+     - Edit mode of the job.  * `UI_LOCKED`: The job is in a locked UI state and cannot be modified. * `EDITABLE`: The job is in an editable state and can be modified.
+  
+   * - `email_notifications`
+     - Map
+     - An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted. See [_](#jobs.<name>.email_notifications).
+  
+   * - `environments`
+     - Sequence
+     - A list of task execution environment specifications that can be referenced by serverless tasks of this job. An environment is required to be present for serverless tasks. For serverless notebook tasks, the environment is accessible in the notebook environment panel. For other serverless tasks, the task environment is required to be specified using environment_key in the task settings. See [_](#jobs.<name>.environments).
+  
+   * - `format`
+     - String
+     - Used to tell what is the format of the job. This field is ignored in Create/Update/Reset calls. When using the Jobs API 2.1 this value is always set to `"MULTI_TASK"`.
+  
+   * - `git_source`
+     - Map
+     - An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.  If `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task.  Note: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job. See [_](#jobs.<name>.git_source).
+  
+   * - `health`
+     - Map
+     - An optional set of health rules that can be defined for this job. See [_](#jobs.<name>.health).
+  
+   * - `job_clusters`
+     - Sequence
+     - A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings. If more than 100 job clusters are available, you can paginate through them using :method:jobs/get. See [_](#jobs.<name>.job_clusters).
+  
+   * - `max_concurrent_runs`
+     - Integer
+     - An optional maximum allowed number of concurrent runs of the job. Set this value if you want to be able to execute multiple runs of the same job concurrently. This is useful for example if you trigger your job on a frequent schedule and want to allow consecutive runs to overlap with each other, or if you want to trigger multiple runs which differ by their input parameters. This setting affects only new runs. For example, suppose the job’s concurrency is 4 and there are 4 concurrent active runs. Then setting the concurrency to 3 won’t kill any of the active runs. However, from then on, new runs are skipped unless there are fewer than 3 active runs. This value cannot exceed 1000. Setting this value to `0` causes all new runs to be skipped.
+  
+   * - `name`
+     - String
+     - An optional name for the job. The maximum length is 4096 bytes in UTF-8 encoding.
+  
+   * - `notification_settings`
+     - Map
+     - Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this job. See [_](#jobs.<name>.notification_settings).
+  
+   * - `parameters`
+     - Sequence
+     - Job-level parameter definitions. See [_](#jobs.<name>.parameters).
+  
+   * - `permissions`
+     - Sequence
+     - See [_](#jobs.<name>.permissions).
+  
+   * - `queue`
+     - Map
+     - The queue settings of the job. See [_](#jobs.<name>.queue).
+  
+   * - `run_as`
+     - Map
+     - Write-only setting. Specifies the user or service principal that the job runs as. If not specified, the job runs as the user who created the job.  Either `user_name` or `service_principal_name` should be specified. If not, an error is thrown. See [_](#jobs.<name>.run_as).
+  
+   * - `schedule`
+     - Map
+     - An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`. See [_](#jobs.<name>.schedule).
+  
+   * - `tags`
+     - Map
+     - A map of tags associated with the job. These are forwarded to the cluster as cluster tags for jobs clusters, and are subject to the same limitations as cluster tags. A maximum of 25 tags can be added to the job.
+  
+   * - `tasks`
+     - Sequence
+     - A list of task specifications to be executed by this job. If more than 100 tasks are available, you can paginate through them using :method:jobs/get. Use the `next_page_token` field at the object root to determine if more results are available. See [_](#jobs.<name>.tasks).
+  
+   * - `timeout_seconds`
+     - Integer
+     - An optional timeout applied to each run of this job. A value of `0` means no timeout.
+  
+   * - `trigger`
+     - Map
+     - A configuration to trigger a run when certain conditions are met. The default behavior is that the job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`. See [_](#jobs.<name>.trigger).
+  
+   * - `webhook_notifications`
+     - Map
+     - A collection of system notification IDs to notify when runs of this job begin or complete. See [_](#jobs.<name>.webhook_notifications).
+  
+  
+**Example**
+  
+The following example defines a job with the resource key `hello-job` with one notebook task:
+
+```yaml
+resources:
+  jobs:
+    hello-job:
+      name: hello-job
+      tasks:
+        - task_key: hello-task
+          notebook_task:
+            notebook_path: ./hello.py
+```
+
+For information about defining job tasks and overriding job settings, see [_](/dev-tools/bundles/job-task-types.md), [_](/dev-tools/bundles/job-task-override.md), and [_](/dev-tools/bundles/cluster-override.md).
+  
+### jobs.<name>.continuous
+  
+**`Type: Map`**
+  
+An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `pause_status`
+     - String
+     - Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED.
+  
+  
+### jobs.<name>.deployment
+  
+**`Type: Map`**
+  
+Deployment information for jobs managed by external sources.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `kind`
+     - String
+     - The kind of deployment that manages the job.  * `BUNDLE`: The job is managed by Databricks Asset Bundle.
+  
+   * - `metadata_file_path`
+     - String
+     - Path of the file that contains deployment metadata.
+  
+  
+### jobs.<name>.email_notifications
+  
+**`Type: Map`**
+  
+An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `no_alert_for_skipped_runs`
+     - Boolean
+     - If true, do not send email to recipients specified in `on_failure` if the run is skipped. This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.
+  
+   * - `on_duration_warning_threshold_exceeded`
+     - Sequence
+     - A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.
+  
+   * - `on_failure`
+     - Sequence
+     - A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent.
+  
+   * - `on_start`
+     - Sequence
+     - A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.
+  
+   * - `on_streaming_backlog_exceeded`
+     - Sequence
+     - A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.
+  
+   * - `on_success`
+     - Sequence
+     - A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.
+  
+  
+### jobs.<name>.environments
+  
+**`Type: Sequence`**
+  
+A list of task execution environment specifications that can be referenced by serverless tasks of this job.
+An environment is required to be present for serverless tasks.
+For serverless notebook tasks, the environment is accessible in the notebook environment panel.
+For other serverless tasks, the task environment is required to be specified using environment_key in the task settings.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `environment_key`
+     - String
+     - The key of an environment. It has to be unique within a job.
+  
+   * - `spec`
+     - Map
+     - The environment entity used to preserve serverless environment side panel and jobs' environment for non-notebook task. In this minimal environment spec, only pip dependencies are supported. See [_](#jobs.<name>.environments.spec).
+  
+  
+### jobs.<name>.environments.spec
+  
+**`Type: Map`**
+  
+The environment entity used to preserve serverless environment side panel and jobs' environment for non-notebook task.
+In this minimal environment spec, only pip dependencies are supported.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `client`
+     - String
+     - Client version used by the environment The client is the user-facing environment of the runtime. Each client comes with a specific set of pre-installed libraries. The version is a string, consisting of the major client version.
+  
+   * - `dependencies`
+     - Sequence
+     - List of pip dependencies, as supported by the version of pip in this environment. Each dependency is a pip requirement file line https://pip.pypa.io/en/stable/reference/requirements-file-format/ Allowed dependency could be <requirement specifier>, <archive url/path>, <local project path>(WSFS or Volumes in Databricks), <vcs project url> E.g. dependencies: ["foo==0.0.1", "-r /Workspace/test/requirements.txt"]
+  
+  
+### jobs.<name>.git_source
+  
+**`Type: Map`**
+  
+An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.
+
+If `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task.
+
+Note: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `git_branch`
+     - String
+     - Name of the branch to be checked out and used by this job. This field cannot be specified in conjunction with git_tag or git_commit.
+  
+   * - `git_commit`
+     - String
+     - Commit to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_tag.
+  
+   * - `git_provider`
+     - String
+     - Unique identifier of the service used to host the Git repository. The value is case insensitive.
+  
+   * - `git_snapshot`
+     - Map
+     - Read-only state of the remote repository at the time the job was run. This field is only included on job runs. See [_](#jobs.<name>.git_source.git_snapshot).
+  
+   * - `git_tag`
+     - String
+     - Name of the tag to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_commit.
+  
+   * - `git_url`
+     - String
+     - URL of the repository to be cloned by this job.
+  
+   * - `job_source`
+     - Map
+     - The source of the job specification in the remote repository when the job is source controlled. See [_](#jobs.<name>.git_source.job_source).
+  
+  
+### jobs.<name>.git_source.git_snapshot
+  
+**`Type: Map`**
+  
+Read-only state of the remote repository at the time the job was run. This field is only included on job runs.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `used_commit`
+     - String
+     - Commit that was used to execute the run. If git_branch was specified, this points to the HEAD of the branch at the time of the run; if git_tag was specified, this points to the commit the tag points to.
+  
+  
+### jobs.<name>.git_source.job_source
+  
+**`Type: Map`**
+  
+The source of the job specification in the remote repository when the job is source controlled.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `dirty_state`
+     - String
+     - Dirty state indicates the job is not fully synced with the job specification in the remote repository.  Possible values are: * `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced. * `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced.
+  
+   * - `import_from_git_branch`
+     - String
+     - Name of the branch which the job is imported from.
+  
+   * - `job_config_path`
+     - String
+     - Path of the job YAML file that contains the job specification.
+  
+  
+### jobs.<name>.health
+  
+**`Type: Map`**
+  
+An optional set of health rules that can be defined for this job.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `rules`
+     - Sequence
+     - See [_](#jobs.<name>.health.rules).
+  
+  
+### jobs.<name>.health.rules
+  
+**`Type: Sequence`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `metric`
+     - String
+     - Specifies the health metric that is being evaluated for a particular health rule.  * `RUN_DURATION_SECONDS`: Expected total time for a run in seconds. * `STREAMING_BACKLOG_BYTES`: An estimate of the maximum bytes of data waiting to be consumed across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_SECONDS`: An estimate of the maximum consumer delay across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all streams. This metric is in Public Preview.
+  
+   * - `op`
+     - String
+     - Specifies the operator used to compare the health metric value with the specified threshold.
+  
+   * - `value`
+     - Integer
+     - Specifies the threshold value that the health metric should obey to satisfy the health rule.
+  
+  
+### jobs.<name>.job_clusters
+  
+**`Type: Sequence`**
+  
+A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings.
+If more than 100 job clusters are available, you can paginate through them using :method:jobs/get.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `job_cluster_key`
+     - String
+     - A unique name for the job cluster. This field is required and must be unique within the job. `JobTaskSettings` may refer to this field to determine which cluster to launch for the task execution.
+  
+   * - `new_cluster`
+     - Map
+     - If new_cluster, a description of a cluster that is created for each task. See [_](#jobs.<name>.job_clusters.new_cluster).
+  
+  
+### jobs.<name>.job_clusters.new_cluster
+  
+**`Type: Map`**
+  
+If new_cluster, a description of a cluster that is created for each task.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `apply_policy_default_values`
+     - Boolean
+     - When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied.
+  
+   * - `autoscale`
+     - Map
+     - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#jobs.<name>.job_clusters.new_cluster.autoscale).
+  
+   * - `autotermination_minutes`
+     - Integer
+     - Automatically terminates the cluster after it is inactive for this time in minutes. If not set, this cluster will not be automatically terminated. If specified, the threshold must be between 10 and 10000 minutes. Users can also set this value to 0 to explicitly disable automatic termination.
+  
+   * - `aws_attributes`
+     - Map
+     - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#jobs.<name>.job_clusters.new_cluster.aws_attributes).
+  
+   * - `azure_attributes`
+     - Map
+     - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#jobs.<name>.job_clusters.new_cluster.azure_attributes).
+  
+   * - `cluster_log_conf`
+     - Map
+     - The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. See [_](#jobs.<name>.job_clusters.new_cluster.cluster_log_conf).
+  
+   * - `cluster_name`
+     - String
+     - Cluster name requested by the user. This doesn't have to be unique. If not specified at creation, the cluster name will be an empty string. 
+  
+   * - `custom_tags`
+     - Map
+     - Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS instances and EBS volumes) with these tags in addition to `default_tags`. Notes:  - Currently, Databricks allows at most 45 custom tags  - Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags
+  
+   * - `data_security_mode`
+     - String
+     - Data security mode decides what data governance model to use when accessing data from a cluster.  The following modes can only be used with `kind`. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`.  The following modes can be used regardless of `kind`. * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.  The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions:  * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. 
+  
+   * - `docker_image`
+     - Map
+     - See [_](#jobs.<name>.job_clusters.new_cluster.docker_image).
+  
+   * - `driver_instance_pool_id`
+     - String
+     - The optional ID of the instance pool for the driver of the cluster belongs. The pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not assigned.
+  
+   * - `driver_node_type_id`
+     - String
+     - The node type of the Spark driver. Note that this field is optional; if unset, the driver node type will be set as the same value as `node_type_id` defined above. 
+  
+   * - `enable_elastic_disk`
+     - Boolean
+     - Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk space when its Spark workers are running low on disk space. This feature requires specific AWS permissions to function correctly - refer to the User Guide for more details.
+  
+   * - `enable_local_disk_encryption`
+     - Boolean
+     - Whether to enable LUKS on cluster VMs' local disks
+  
+   * - `gcp_attributes`
+     - Map
+     - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#jobs.<name>.job_clusters.new_cluster.gcp_attributes).
+  
+   * - `init_scripts`
+     - Sequence
+     - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `<destination>/<cluster-ID>/init_scripts`. See [_](#jobs.<name>.job_clusters.new_cluster.init_scripts).
+  
+   * - `instance_pool_id`
+     - String
+     - The optional ID of the instance pool to which the cluster belongs.
+  
+   * - `is_single_node`
+     - Boolean
+     - This field can only be used with `kind`.  When set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, and `num_workers` 
+  
+   * - `kind`
+     - String
+     - 
+  
+   * - `node_type_id`
+     - String
+     - This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or compute intensive workloads. A list of available node types can be retrieved by using the :method:clusters/listNodeTypes API call. 
+  
+   * - `num_workers`
+     - Integer
+     - Number of worker nodes that this cluster should have. A cluster has one Spark Driver and `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.  Note: When reading the properties of a cluster, this field reflects the desired number of workers rather than the actual current number of workers. For instance, if a cluster is resized from 5 to 10 workers, this field will immediately be updated to reflect the target size of 10 workers, whereas the workers listed in `spark_info` will gradually increase from 5 to 10 as the new nodes are provisioned.
+  
+   * - `policy_id`
+     - String
+     - The ID of the cluster policy used to create the cluster if applicable.
+  
+   * - `runtime_engine`
+     - String
+     - Determines the cluster's runtime engine, either standard or Photon.  This field is not compatible with legacy `spark_version` values that contain `-photon-`. Remove `-photon-` from the `spark_version` and set `runtime_engine` to `PHOTON`.  If left unspecified, the runtime engine defaults to standard unless the spark_version contains -photon-, in which case Photon will be used. 
+  
+   * - `single_user_name`
+     - String
+     - Single user name if data_security_mode is `SINGLE_USER`
+  
+   * - `spark_conf`
+     - Map
+     - An object containing a set of optional, user-specified Spark configuration key-value pairs. Users can also pass in a string of extra JVM options to the driver and the executors via `spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively. 
+  
+   * - `spark_env_vars`
+     - Map
+     - An object containing a set of optional, user-specified environment variable key-value pairs. Please note that key-value pair of the form (X,Y) will be exported as is (i.e., `export X='Y'`) while launching the driver and workers.  In order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending them to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all default databricks managed environmental variables are included as well.  Example Spark environment variables: `{"SPARK_WORKER_MEMORY": "28000m", "SPARK_LOCAL_DIRS": "/local_disk0"}` or `{"SPARK_DAEMON_JAVA_OPTS": "$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true"}`
+  
+   * - `spark_version`
+     - String
+     - The Spark version of the cluster, e.g. `3.3.x-scala2.11`. A list of available Spark versions can be retrieved by using the :method:clusters/sparkVersions API call. 
+  
+   * - `ssh_public_keys`
+     - Sequence
+     - SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified.
+  
+   * - `use_ml_runtime`
+     - Boolean
+     - This field can only be used with `kind`.  `effective_spark_version` is determined by `spark_version` (DBR release), this field `use_ml_runtime`, and whether `node_type_id` is gpu node or not. 
+  
+   * - `workload_type`
+     - Map
+     - See [_](#jobs.<name>.job_clusters.new_cluster.workload_type).
+  
+  
+### jobs.<name>.job_clusters.new_cluster.autoscale
+  
+**`Type: Map`**
+  
+Parameters needed in order to automatically scale clusters up and down based on load.
+Note: autoscaling works best with DB runtime versions 3.0 or later.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `max_workers`
+     - Integer
+     - The maximum number of workers to which the cluster can scale up when overloaded. Note that `max_workers` must be strictly greater than `min_workers`.
+  
+   * - `min_workers`
+     - Integer
+     - The minimum number of workers to which the cluster can scale down when underutilized. It is also the initial number of workers the cluster will have after creation.
+  
+  
+### jobs.<name>.job_clusters.new_cluster.aws_attributes
+  
+**`Type: Map`**
+  
+Attributes related to clusters running on Amazon Web Services.
+If not specified at cluster creation, a set of default values will be used.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `availability`
+     - String
+     - Availability type used for all subsequent nodes past the `first_on_demand` ones.  Note: If `first_on_demand` is zero, this availability type will be used for the entire cluster. 
+  
+   * - `ebs_volume_count`
+     - Integer
+     - The number of volumes launched for each instance. Users can choose up to 10 volumes. This feature is only enabled for supported node types. Legacy node types cannot specify custom EBS volumes. For node types with no instance store, at least one EBS volume needs to be specified; otherwise, cluster creation will fail.  These EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc. Instance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc.  If EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for scratch storage because heterogenously sized scratch devices can lead to inefficient disk utilization. If no EBS volumes are attached, Databricks will configure Spark to use instance store volumes.  Please note that if EBS volumes are specified, then the Spark configuration `spark.local.dir` will be overridden.
+  
+   * - `ebs_volume_iops`
+     - Integer
+     - If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used.
+  
+   * - `ebs_volume_size`
+     - Integer
+     - The size of each EBS volume (in GiB) launched for each instance. For general purpose SSD, this value must be within the range 100 - 4096. For throughput optimized HDD, this value must be within the range 500 - 4096.
+  
+   * - `ebs_volume_throughput`
+     - Integer
+     - If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used.
+  
+   * - `ebs_volume_type`
+     - String
+     - The type of EBS volumes that will be launched with this cluster.
+  
+   * - `first_on_demand`
+     - Integer
+     - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. If this value is greater than 0, the cluster driver node in particular will be placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster.
+  
+   * - `instance_profile_arn`
+     - String
+     - Nodes for this cluster will only be placed on AWS instances with this instance profile. If ommitted, nodes will be placed on instances without an IAM instance profile. The instance profile must have previously been added to the Databricks environment by an account administrator.  This feature may only be available to certain customer plans.  If this field is ommitted, we will pull in the default from the conf if it exists.
+  
+   * - `spot_bid_price_percent`
+     - Integer
+     - The bid price for AWS spot instances, as a percentage of the corresponding instance type's on-demand price. For example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot instance, then the bid price is half of the price of on-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice the price of on-demand `r3.xlarge` instances. If not specified, the default value is 100. When spot instances are requested for this cluster, only spot instances whose bid price percentage matches this field will be considered. Note that, for safety, we enforce this field to be no more than 10000.  The default value and documentation here should be kept consistent with CommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent.
+  
+   * - `zone_id`
+     - String
+     - Identifier for the availability zone/datacenter in which the cluster resides. This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. This is an optional field at cluster creation, and if not specified, a default zone will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. The list of available zones as well as the default value can be found by using the `List Zones` method.
+  
+  
+### jobs.<name>.job_clusters.new_cluster.azure_attributes
+  
+**`Type: Map`**
+  
+Attributes related to clusters running on Microsoft Azure.
+If not specified at cluster creation, a set of default values will be used.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `availability`
+     - String
+     - Availability type used for all subsequent nodes past the `first_on_demand` ones. Note: If `first_on_demand` is zero (which only happens on pool clusters), this availability type will be used for the entire cluster.
+  
+   * - `first_on_demand`
+     - Integer
+     - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster.
+  
+   * - `log_analytics_info`
+     - Map
+     - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#jobs.<name>.job_clusters.new_cluster.azure_attributes.log_analytics_info).
+  
+   * - `spot_bid_max_price`
+     - Any
+     - The max bid price to be used for Azure spot instances. The Max price for the bid cannot be higher than the on-demand price of the instance. If not specified, the default value is -1, which specifies that the instance cannot be evicted on the basis of price, and only on the basis of availability. Further, the value should > 0 or -1.
+  
+  
+### jobs.<name>.job_clusters.new_cluster.azure_attributes.log_analytics_info
+  
+**`Type: Map`**
+  
+Defines values necessary to configure and run Azure Log Analytics agent
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `log_analytics_primary_key`
+     - String
+     - <needs content added>
+  
+   * - `log_analytics_workspace_id`
+     - String
+     - <needs content added>
+  
+  
+### jobs.<name>.job_clusters.new_cluster.cluster_log_conf
+  
+**`Type: Map`**
+  
+The configuration for delivering spark logs to a long-term storage destination.
+Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified
+for one cluster. If the conf is given, the logs will be delivered to the destination every
+`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while
+the destination of executor logs is `$destination/$clusterId/executor`.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `dbfs`
+     - Map
+     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobs.<name>.job_clusters.new_cluster.cluster_log_conf.dbfs).
+  
+   * - `s3`
+     - Map
+     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobs.<name>.job_clusters.new_cluster.cluster_log_conf.s3).
+  
+  
+### jobs.<name>.job_clusters.new_cluster.cluster_log_conf.dbfs
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - dbfs destination, e.g. `dbfs:/my/path`
+  
+  
+### jobs.<name>.job_clusters.new_cluster.cluster_log_conf.s3
+  
+**`Type: Map`**
+  
+destination and either the region or endpoint need to be provided. e.g.
+`{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }`
+Cluster iam role is used to access s3, please make sure the cluster iam role in
+`instance_profile_arn` has permission to write data to the s3 destination.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `canned_acl`
+     - String
+     - (Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`. If `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on the destination bucket and prefix. The full list of possible canned acl can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl. Please also note that by default only the object owner gets full controls. If you are using cross account role for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to read the logs.
+  
+   * - `destination`
+     - String
+     - S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using cluster iam role, please make sure you set cluster iam role and the role has write access to the destination. Please also note that you cannot use AWS keys to deliver logs.
+  
+   * - `enable_encryption`
+     - Boolean
+     - (Optional) Flag to enable server side encryption, `false` by default.
+  
+   * - `encryption_type`
+     - String
+     - (Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when encryption is enabled and the default type is `sse-s3`.
+  
+   * - `endpoint`
+     - String
+     - S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
+  
+   * - `kms_key`
+     - String
+     - (Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`.
+  
+   * - `region`
+     - String
+     - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
+  
+  
+### jobs.<name>.job_clusters.new_cluster.docker_image
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `basic_auth`
+     - Map
+     - See [_](#jobs.<name>.job_clusters.new_cluster.docker_image.basic_auth).
+  
+   * - `url`
+     - String
+     - URL of the docker image.
+  
+  
+### jobs.<name>.job_clusters.new_cluster.docker_image.basic_auth
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `password`
+     - String
+     - Password of the user
+  
+   * - `username`
+     - String
+     - Name of the user
+  
+  
+### jobs.<name>.job_clusters.new_cluster.gcp_attributes
+  
+**`Type: Map`**
+  
+Attributes related to clusters running on Google Cloud Platform.
+If not specified at cluster creation, a set of default values will be used.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `availability`
+     - String
+     - This field determines whether the instance pool will contain preemptible VMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable.
+  
+   * - `boot_disk_size`
+     - Integer
+     - boot disk size in GB
+  
+   * - `google_service_account`
+     - String
+     - If provided, the cluster will impersonate the google service account when accessing gcloud services (like GCS). The google service account must have previously been added to the Databricks environment by an account administrator.
+  
+   * - `local_ssd_count`
+     - Integer
+     - If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type.
+  
+   * - `use_preemptible_executors`
+     - Boolean
+     - This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default). Note: Soon to be deprecated, use the availability field instead.
+  
+   * - `zone_id`
+     - String
+     - Identifier for the availability zone in which the cluster resides. This can be one of the following: - "HA" => High availability, spread nodes across availability zones for a Databricks deployment region [default] - "AUTO" => Databricks picks an availability zone to schedule the cluster on. - A GCP availability zone => Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones.
+  
+  
+### jobs.<name>.job_clusters.new_cluster.init_scripts
+  
+**`Type: Sequence`**
+  
+The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `<destination>/<cluster-ID>/init_scripts`.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `abfss`
+     - Map
+     - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>" } }. See [_](#jobs.<name>.job_clusters.new_cluster.init_scripts.abfss).
+  
+   * - `dbfs`
+     - Map
+     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobs.<name>.job_clusters.new_cluster.init_scripts.dbfs).
+  
+   * - `file`
+     - Map
+     - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#jobs.<name>.job_clusters.new_cluster.init_scripts.file).
+  
+   * - `gcs`
+     - Map
+     - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#jobs.<name>.job_clusters.new_cluster.init_scripts.gcs).
+  
+   * - `s3`
+     - Map
+     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobs.<name>.job_clusters.new_cluster.init_scripts.s3).
+  
+   * - `volumes`
+     - Map
+     - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#jobs.<name>.job_clusters.new_cluster.init_scripts.volumes).
+  
+   * - `workspace`
+     - Map
+     - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#jobs.<name>.job_clusters.new_cluster.init_scripts.workspace).
+  
+  
+### jobs.<name>.job_clusters.new_cluster.init_scripts.abfss
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "abfss" : { "destination" : "abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>" } }
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - abfss destination, e.g. `abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>`.
+  
+  
+### jobs.<name>.job_clusters.new_cluster.init_scripts.dbfs
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - dbfs destination, e.g. `dbfs:/my/path`
+  
+  
+### jobs.<name>.job_clusters.new_cluster.init_scripts.file
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "file" : { "destination" : "file:/my/local/file.sh" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - local file destination, e.g. `file:/my/local/file.sh`
+  
+  
+### jobs.<name>.job_clusters.new_cluster.init_scripts.gcs
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - GCS destination/URI, e.g. `gs://my-bucket/some-prefix`
+  
+  
+### jobs.<name>.job_clusters.new_cluster.init_scripts.s3
+  
+**`Type: Map`**
+  
+destination and either the region or endpoint need to be provided. e.g.
+`{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }`
+Cluster iam role is used to access s3, please make sure the cluster iam role in
+`instance_profile_arn` has permission to write data to the s3 destination.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `canned_acl`
+     - String
+     - (Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`. If `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on the destination bucket and prefix. The full list of possible canned acl can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl. Please also note that by default only the object owner gets full controls. If you are using cross account role for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to read the logs.
+  
+   * - `destination`
+     - String
+     - S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using cluster iam role, please make sure you set cluster iam role and the role has write access to the destination. Please also note that you cannot use AWS keys to deliver logs.
+  
+   * - `enable_encryption`
+     - Boolean
+     - (Optional) Flag to enable server side encryption, `false` by default.
+  
+   * - `encryption_type`
+     - String
+     - (Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when encryption is enabled and the default type is `sse-s3`.
+  
+   * - `endpoint`
+     - String
+     - S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
+  
+   * - `kms_key`
+     - String
+     - (Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`.
+  
+   * - `region`
+     - String
+     - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
+  
+  
+### jobs.<name>.job_clusters.new_cluster.init_scripts.volumes
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`
+  
+  
+### jobs.<name>.job_clusters.new_cluster.init_scripts.workspace
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`
+  
+  
+### jobs.<name>.job_clusters.new_cluster.workload_type
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `clients`
+     - Map
+     -  defined what type of clients can use the cluster. E.g. Notebooks, Jobs. See [_](#jobs.<name>.job_clusters.new_cluster.workload_type.clients).
+  
+  
+### jobs.<name>.job_clusters.new_cluster.workload_type.clients
+  
+**`Type: Map`**
+  
+ defined what type of clients can use the cluster. E.g. Notebooks, Jobs
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `jobs`
+     - Boolean
+     - With jobs set, the cluster can be used for jobs
+  
+   * - `notebooks`
+     - Boolean
+     - With notebooks set, this cluster can be used for notebooks
+  
+  
+### jobs.<name>.notification_settings
+  
+**`Type: Map`**
+  
+Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this job.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `no_alert_for_canceled_runs`
+     - Boolean
+     - If true, do not send notifications to recipients specified in `on_failure` if the run is canceled.
+  
+   * - `no_alert_for_skipped_runs`
+     - Boolean
+     - If true, do not send notifications to recipients specified in `on_failure` if the run is skipped.
+  
+  
+### jobs.<name>.parameters
+  
+**`Type: Sequence`**
+  
+Job-level parameter definitions
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `default`
+     - String
+     - Default value of the parameter.
+  
+   * - `name`
+     - String
+     - The name of the defined parameter. May only contain alphanumeric characters, `_`, `-`, and `.`
+  
+  
+### jobs.<name>.permissions
+  
+**`Type: Sequence`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `group_name`
+     - String
+     - The name of the group that has the permission set in level.
+  
+   * - `level`
+     - String
+     - The allowed permission for user, group, service principal defined for this permission.
+  
+   * - `service_principal_name`
+     - String
+     - The name of the service principal that has the permission set in level.
+  
+   * - `user_name`
+     - String
+     - The name of the user that has the permission set in level.
+  
+  
+### jobs.<name>.queue
+  
+**`Type: Map`**
+  
+The queue settings of the job.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `enabled`
+     - Boolean
+     - If true, enable queueing for the job. This is a required field.
+  
+  
+### jobs.<name>.run_as
+  
+**`Type: Map`**
+  
+Write-only setting. Specifies the user or service principal that the job runs as. If not specified, the job runs as the user who created the job.
+
+Either `user_name` or `service_principal_name` should be specified. If not, an error is thrown.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `service_principal_name`
+     - String
+     - The application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.
+  
+   * - `user_name`
+     - String
+     - The email of an active workspace user. Non-admin users can only set this field to their own email.
+  
+  
+### jobs.<name>.schedule
+  
+**`Type: Map`**
+  
+An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `pause_status`
+     - String
+     - Indicate whether this schedule is paused or not.
+  
+   * - `quartz_cron_expression`
+     - String
+     - A Cron expression using Quartz syntax that describes the schedule for a job. See [Cron Trigger](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) for details. This field is required.
+  
+   * - `timezone_id`
+     - String
+     - A Java timezone ID. The schedule for a job is resolved with respect to this timezone. See [Java TimeZone](https://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html) for details. This field is required.
+  
+  
+### jobs.<name>.tasks
+  
+**`Type: Sequence`**
+  
+A list of task specifications to be executed by this job.
+If more than 100 tasks are available, you can paginate through them using :method:jobs/get. Use the `next_page_token` field at the object root to determine if more results are available.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `clean_rooms_notebook_task`
+     - Map
+     - The task runs a [clean rooms](https://docs.databricks.com/en/clean-rooms/index.html) notebook when the `clean_rooms_notebook_task` field is present. See [_](#jobs.<name>.tasks.clean_rooms_notebook_task).
+  
+   * - `condition_task`
+     - Map
+     - The task evaluates a condition that can be used to control the execution of other tasks when the `condition_task` field is present. The condition task does not require a cluster to execute and does not support retries or notifications. See [_](#jobs.<name>.tasks.condition_task).
+  
+   * - `dbt_task`
+     - Map
+     - The task runs one or more dbt commands when the `dbt_task` field is present. The dbt task requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse. See [_](#jobs.<name>.tasks.dbt_task).
+  
+   * - `depends_on`
+     - Sequence
+     - An optional array of objects specifying the dependency graph of the task. All tasks specified in this field must complete before executing this task. The task will run only if the `run_if` condition is true. The key is `task_key`, and the value is the name assigned to the dependent task. See [_](#jobs.<name>.tasks.depends_on).
+  
+   * - `description`
+     - String
+     - An optional description for this task.
+  
+   * - `disable_auto_optimization`
+     - Boolean
+     - An option to disable auto optimization in serverless
+  
+   * - `email_notifications`
+     - Map
+     - An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails. See [_](#jobs.<name>.tasks.email_notifications).
+  
+   * - `environment_key`
+     - String
+     - The key that references an environment spec in a job. This field is required for Python script, Python wheel and dbt tasks when using serverless compute.
+  
+   * - `existing_cluster_id`
+     - String
+     - If existing_cluster_id, the ID of an existing cluster that is used for all runs. When running jobs or tasks on an existing cluster, you may need to manually restart the cluster if it stops responding. We suggest running jobs and tasks on new clusters for greater reliability
+  
+   * - `for_each_task`
+     - Map
+     - The task executes a nested task for every input provided when the `for_each_task` field is present. See [_](#jobs.<name>.tasks.for_each_task).
+  
+   * - `health`
+     - Map
+     - An optional set of health rules that can be defined for this job. See [_](#jobs.<name>.tasks.health).
+  
+   * - `job_cluster_key`
+     - String
+     - If job_cluster_key, this task is executed reusing the cluster specified in `job.settings.job_clusters`.
+  
+   * - `libraries`
+     - Sequence
+     - An optional list of libraries to be installed on the cluster. The default value is an empty list. See [_](#jobs.<name>.tasks.libraries).
+  
+   * - `max_retries`
+     - Integer
+     - An optional maximum number of times to retry an unsuccessful run. A run is considered to be unsuccessful if it completes with the `FAILED` result_state or `INTERNAL_ERROR` `life_cycle_state`. The value `-1` means to retry indefinitely and the value `0` means to never retry.
+  
+   * - `min_retry_interval_millis`
+     - Integer
+     - An optional minimal interval in milliseconds between the start of the failed run and the subsequent retry run. The default behavior is that unsuccessful runs are immediately retried.
+  
+   * - `new_cluster`
+     - Map
+     - If new_cluster, a description of a new cluster that is created for each run. See [_](#jobs.<name>.tasks.new_cluster).
+  
+   * - `notebook_task`
+     - Map
+     - The task runs a notebook when the `notebook_task` field is present. See [_](#jobs.<name>.tasks.notebook_task).
+  
+   * - `notification_settings`
+     - Map
+     - Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this task. See [_](#jobs.<name>.tasks.notification_settings).
+  
+   * - `pipeline_task`
+     - Map
+     - The task triggers a pipeline update when the `pipeline_task` field is present. Only pipelines configured to use triggered more are supported. See [_](#jobs.<name>.tasks.pipeline_task).
+  
+   * - `python_wheel_task`
+     - Map
+     - The task runs a Python wheel when the `python_wheel_task` field is present. See [_](#jobs.<name>.tasks.python_wheel_task).
+  
+   * - `retry_on_timeout`
+     - Boolean
+     - An optional policy to specify whether to retry a job when it times out. The default behavior is to not retry on timeout.
+  
+   * - `run_if`
+     - String
+     - An optional value specifying the condition determining whether the task is run once its dependencies have been completed.  * `ALL_SUCCESS`: All dependencies have executed and succeeded * `AT_LEAST_ONE_SUCCESS`: At least one dependency has succeeded * `NONE_FAILED`: None of the dependencies have failed and at least one was executed * `ALL_DONE`: All dependencies have been completed * `AT_LEAST_ONE_FAILED`: At least one dependency failed * `ALL_FAILED`: ALl dependencies have failed
+  
+   * - `run_job_task`
+     - Map
+     - The task triggers another job when the `run_job_task` field is present. See [_](#jobs.<name>.tasks.run_job_task).
+  
+   * - `spark_jar_task`
+     - Map
+     - The task runs a JAR when the `spark_jar_task` field is present. See [_](#jobs.<name>.tasks.spark_jar_task).
+  
+   * - `spark_python_task`
+     - Map
+     - The task runs a Python file when the `spark_python_task` field is present. See [_](#jobs.<name>.tasks.spark_python_task).
+  
+   * - `spark_submit_task`
+     - Map
+     - (Legacy) The task runs the spark-submit script when the `spark_submit_task` field is present. This task can run only on new clusters and is not compatible with serverless compute.  In the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark configurations.  `master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you _cannot_ specify them in parameters.  By default, the Spark submit job uses all available memory (excluding reserved memory for Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value to leave some room for off-heap usage.  The `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths. See [_](#jobs.<name>.tasks.spark_submit_task).
+  
+   * - `sql_task`
+     - Map
+     - The task runs a SQL query or file, or it refreshes a SQL alert or a legacy SQL dashboard when the `sql_task` field is present. See [_](#jobs.<name>.tasks.sql_task).
+  
+   * - `task_key`
+     - String
+     - A unique name for the task. This field is used to refer to this task from other tasks. This field is required and must be unique within its parent job. On Update or Reset, this field is used to reference the tasks to be updated or reset.
+  
+   * - `timeout_seconds`
+     - Integer
+     - An optional timeout applied to each run of this job task. A value of `0` means no timeout.
+  
+   * - `webhook_notifications`
+     - Map
+     - A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications. See [_](#jobs.<name>.tasks.webhook_notifications).
+  
+  
+### jobs.<name>.tasks.clean_rooms_notebook_task
+  
+**`Type: Map`**
+  
+The task runs a [clean rooms](https://docs.databricks.com/en/clean-rooms/index.html) notebook
+when the `clean_rooms_notebook_task` field is present.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `clean_room_name`
+     - String
+     - The clean room that the notebook belongs to.
+  
+   * - `etag`
+     - String
+     - Checksum to validate the freshness of the notebook resource (i.e. the notebook being run is the latest version). It can be fetched by calling the :method:cleanroomassets/get API.
+  
+   * - `notebook_base_parameters`
+     - Map
+     - Base parameters to be used for the clean room notebook job.
+  
+   * - `notebook_name`
+     - String
+     - Name of the notebook being run.
+  
+  
+### jobs.<name>.tasks.condition_task
+  
+**`Type: Map`**
+  
+The task evaluates a condition that can be used to control the execution of other tasks when the `condition_task` field is present.
+The condition task does not require a cluster to execute and does not support retries or notifications.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `left`
+     - String
+     - The left operand of the condition task. Can be either a string value or a job state or parameter reference.
+  
+   * - `op`
+     - String
+     - * `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`. * `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” >= “12”` will evaluate to `true`, `“10.0” >= “12”` will evaluate to `false`.  The boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison.
+  
+   * - `right`
+     - String
+     - The right operand of the condition task. Can be either a string value or a job state or parameter reference.
+  
+  
+### jobs.<name>.tasks.dbt_task
+  
+**`Type: Map`**
+  
+The task runs one or more dbt commands when the `dbt_task` field is present. The dbt task requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `catalog`
+     - String
+     - Optional name of the catalog to use. The value is the top level in the 3-level namespace of Unity Catalog (catalog / schema / relation). The catalog value can only be specified if a warehouse_id is specified. Requires dbt-databricks >= 1.1.1.
+  
+   * - `commands`
+     - Sequence
+     - A list of dbt commands to execute. All commands must start with `dbt`. This parameter must not be empty. A maximum of up to 10 commands can be provided.
+  
+   * - `profiles_directory`
+     - String
+     - Optional (relative) path to the profiles directory. Can only be specified if no warehouse_id is specified. If no warehouse_id is specified and this folder is unset, the root directory is used.
+  
+   * - `project_directory`
+     - String
+     - Path to the project directory. Optional for Git sourced tasks, in which case if no value is provided, the root of the Git repository is used.
+  
+   * - `schema`
+     - String
+     - Optional schema to write to. This parameter is only used when a warehouse_id is also provided. If not provided, the `default` schema is used.
+  
+   * - `source`
+     - String
+     - Optional location type of the project directory. When set to `WORKSPACE`, the project will be retrieved from the local Databricks workspace. When set to `GIT`, the project will be retrieved from a Git repository defined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.  * `WORKSPACE`: Project is located in Databricks workspace. * `GIT`: Project is located in cloud Git provider.
+  
+   * - `warehouse_id`
+     - String
+     - ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the profile and connection details to dbt. It can be overridden on a per-command basis by using the `--profiles-dir` command line argument.
+  
+  
+### jobs.<name>.tasks.depends_on
+  
+**`Type: Sequence`**
+  
+An optional array of objects specifying the dependency graph of the task. All tasks specified in this field must complete before executing this task. The task will run only if the `run_if` condition is true.
+The key is `task_key`, and the value is the name assigned to the dependent task.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `outcome`
+     - String
+     - Can only be specified on condition task dependencies. The outcome of the dependent task that must be met for this task to run.
+  
+   * - `task_key`
+     - String
+     - The name of the task this task depends on.
+  
+  
+### jobs.<name>.tasks.email_notifications
+  
+**`Type: Map`**
+  
+An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `no_alert_for_skipped_runs`
+     - Boolean
+     - If true, do not send email to recipients specified in `on_failure` if the run is skipped. This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.
+  
+   * - `on_duration_warning_threshold_exceeded`
+     - Sequence
+     - A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.
+  
+   * - `on_failure`
+     - Sequence
+     - A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent.
+  
+   * - `on_start`
+     - Sequence
+     - A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.
+  
+   * - `on_streaming_backlog_exceeded`
+     - Sequence
+     - A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.
+  
+   * - `on_success`
+     - Sequence
+     - A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.
+  
+  
+### jobs.<name>.tasks.for_each_task
+  
+**`Type: Map`**
+  
+The task executes a nested task for every input provided when the `for_each_task` field is present.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `concurrency`
+     - Integer
+     - An optional maximum allowed number of concurrent runs of the task. Set this value if you want to be able to execute multiple runs of the task concurrently.
+  
+   * - `inputs`
+     - String
+     - Array for task to iterate on. This can be a JSON string or a reference to an array parameter.
+  
+   * - `task`
+     - Map
+     - Configuration for the task that will be run for each element in the array
+  
+  
+### jobs.<name>.tasks.health
+  
+**`Type: Map`**
+  
+An optional set of health rules that can be defined for this job.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `rules`
+     - Sequence
+     - See [_](#jobs.<name>.tasks.health.rules).
+  
+  
+### jobs.<name>.tasks.health.rules
+  
+**`Type: Sequence`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `metric`
+     - String
+     - Specifies the health metric that is being evaluated for a particular health rule.  * `RUN_DURATION_SECONDS`: Expected total time for a run in seconds. * `STREAMING_BACKLOG_BYTES`: An estimate of the maximum bytes of data waiting to be consumed across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_SECONDS`: An estimate of the maximum consumer delay across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all streams. This metric is in Public Preview.
+  
+   * - `op`
+     - String
+     - Specifies the operator used to compare the health metric value with the specified threshold.
+  
+   * - `value`
+     - Integer
+     - Specifies the threshold value that the health metric should obey to satisfy the health rule.
+  
+  
+### jobs.<name>.tasks.libraries
+  
+**`Type: Sequence`**
+  
+An optional list of libraries to be installed on the cluster.
+The default value is an empty list.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `cran`
+     - Map
+     - Specification of a CRAN library to be installed as part of the library. See [_](#jobs.<name>.tasks.libraries.cran).
+  
+   * - `egg`
+     - String
+     - Deprecated. URI of the egg library to install. Installing Python egg files is deprecated and is not supported in Databricks Runtime 14.0 and above.
+  
+   * - `jar`
+     - String
+     - URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs. For example: `{ "jar": "/Workspace/path/to/library.jar" }`, `{ "jar" : "/Volumes/path/to/library.jar" }` or `{ "jar": "s3://my-bucket/library.jar" }`. If S3 is used, please make sure the cluster has read access on the library. You may need to launch the cluster with an IAM role to access the S3 URI.
+  
+   * - `maven`
+     - Map
+     - Specification of a maven library to be installed. For example: `{ "coordinates": "org.jsoup:jsoup:1.7.2" }`. See [_](#jobs.<name>.tasks.libraries.maven).
+  
+   * - `pypi`
+     - Map
+     - Specification of a PyPi library to be installed. For example: `{ "package": "simplejson" }`. See [_](#jobs.<name>.tasks.libraries.pypi).
+  
+   * - `requirements`
+     - String
+     - URI of the requirements.txt file to install. Only Workspace paths and Unity Catalog Volumes paths are supported. For example: `{ "requirements": "/Workspace/path/to/requirements.txt" }` or `{ "requirements" : "/Volumes/path/to/requirements.txt" }`
+  
+   * - `whl`
+     - String
+     - URI of the wheel library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs. For example: `{ "whl": "/Workspace/path/to/library.whl" }`, `{ "whl" : "/Volumes/path/to/library.whl" }` or `{ "whl": "s3://my-bucket/library.whl" }`. If S3 is used, please make sure the cluster has read access on the library. You may need to launch the cluster with an IAM role to access the S3 URI.
+  
+  
+### jobs.<name>.tasks.libraries.cran
+  
+**`Type: Map`**
+  
+Specification of a CRAN library to be installed as part of the library
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `package`
+     - String
+     - The name of the CRAN package to install.
+  
+   * - `repo`
+     - String
+     - The repository where the package can be found. If not specified, the default CRAN repo is used.
+  
+  
+### jobs.<name>.tasks.libraries.maven
+  
+**`Type: Map`**
+  
+Specification of a maven library to be installed. For example:
+`{ "coordinates": "org.jsoup:jsoup:1.7.2" }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `coordinates`
+     - String
+     - Gradle-style maven coordinates. For example: "org.jsoup:jsoup:1.7.2".
+  
+   * - `exclusions`
+     - Sequence
+     - List of dependences to exclude. For example: `["slf4j:slf4j", "*:hadoop-client"]`.  Maven dependency exclusions: https://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html.
+  
+   * - `repo`
+     - String
+     - Maven repo to install the Maven package from. If omitted, both Maven Central Repository and Spark Packages are searched.
+  
+  
+### jobs.<name>.tasks.libraries.pypi
+  
+**`Type: Map`**
+  
+Specification of a PyPi library to be installed. For example:
+`{ "package": "simplejson" }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `package`
+     - String
+     - The name of the pypi package to install. An optional exact version specification is also supported. Examples: "simplejson" and "simplejson==3.8.0".
+  
+   * - `repo`
+     - String
+     - The repository where the package can be found. If not specified, the default pip index is used.
+  
+  
+### jobs.<name>.tasks.new_cluster
+  
+**`Type: Map`**
+  
+If new_cluster, a description of a new cluster that is created for each run.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `apply_policy_default_values`
+     - Boolean
+     - When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied.
+  
+   * - `autoscale`
+     - Map
+     - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#jobs.<name>.tasks.new_cluster.autoscale).
+  
+   * - `autotermination_minutes`
+     - Integer
+     - Automatically terminates the cluster after it is inactive for this time in minutes. If not set, this cluster will not be automatically terminated. If specified, the threshold must be between 10 and 10000 minutes. Users can also set this value to 0 to explicitly disable automatic termination.
+  
+   * - `aws_attributes`
+     - Map
+     - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#jobs.<name>.tasks.new_cluster.aws_attributes).
+  
+   * - `azure_attributes`
+     - Map
+     - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#jobs.<name>.tasks.new_cluster.azure_attributes).
+  
+   * - `cluster_log_conf`
+     - Map
+     - The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. See [_](#jobs.<name>.tasks.new_cluster.cluster_log_conf).
+  
+   * - `cluster_name`
+     - String
+     - Cluster name requested by the user. This doesn't have to be unique. If not specified at creation, the cluster name will be an empty string. 
+  
+   * - `custom_tags`
+     - Map
+     - Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS instances and EBS volumes) with these tags in addition to `default_tags`. Notes:  - Currently, Databricks allows at most 45 custom tags  - Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags
+  
+   * - `data_security_mode`
+     - String
+     - Data security mode decides what data governance model to use when accessing data from a cluster.  The following modes can only be used with `kind`. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`.  The following modes can be used regardless of `kind`. * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.  The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions:  * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. 
+  
+   * - `docker_image`
+     - Map
+     - See [_](#jobs.<name>.tasks.new_cluster.docker_image).
+  
+   * - `driver_instance_pool_id`
+     - String
+     - The optional ID of the instance pool for the driver of the cluster belongs. The pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not assigned.
+  
+   * - `driver_node_type_id`
+     - String
+     - The node type of the Spark driver. Note that this field is optional; if unset, the driver node type will be set as the same value as `node_type_id` defined above. 
+  
+   * - `enable_elastic_disk`
+     - Boolean
+     - Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk space when its Spark workers are running low on disk space. This feature requires specific AWS permissions to function correctly - refer to the User Guide for more details.
+  
+   * - `enable_local_disk_encryption`
+     - Boolean
+     - Whether to enable LUKS on cluster VMs' local disks
+  
+   * - `gcp_attributes`
+     - Map
+     - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#jobs.<name>.tasks.new_cluster.gcp_attributes).
+  
+   * - `init_scripts`
+     - Sequence
+     - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `<destination>/<cluster-ID>/init_scripts`. See [_](#jobs.<name>.tasks.new_cluster.init_scripts).
+  
+   * - `instance_pool_id`
+     - String
+     - The optional ID of the instance pool to which the cluster belongs.
+  
+   * - `is_single_node`
+     - Boolean
+     - This field can only be used with `kind`.  When set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, and `num_workers` 
+  
+   * - `kind`
+     - String
+     - 
+  
+   * - `node_type_id`
+     - String
+     - This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or compute intensive workloads. A list of available node types can be retrieved by using the :method:clusters/listNodeTypes API call. 
+  
+   * - `num_workers`
+     - Integer
+     - Number of worker nodes that this cluster should have. A cluster has one Spark Driver and `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.  Note: When reading the properties of a cluster, this field reflects the desired number of workers rather than the actual current number of workers. For instance, if a cluster is resized from 5 to 10 workers, this field will immediately be updated to reflect the target size of 10 workers, whereas the workers listed in `spark_info` will gradually increase from 5 to 10 as the new nodes are provisioned.
+  
+   * - `policy_id`
+     - String
+     - The ID of the cluster policy used to create the cluster if applicable.
+  
+   * - `runtime_engine`
+     - String
+     - Determines the cluster's runtime engine, either standard or Photon.  This field is not compatible with legacy `spark_version` values that contain `-photon-`. Remove `-photon-` from the `spark_version` and set `runtime_engine` to `PHOTON`.  If left unspecified, the runtime engine defaults to standard unless the spark_version contains -photon-, in which case Photon will be used. 
+  
+   * - `single_user_name`
+     - String
+     - Single user name if data_security_mode is `SINGLE_USER`
+  
+   * - `spark_conf`
+     - Map
+     - An object containing a set of optional, user-specified Spark configuration key-value pairs. Users can also pass in a string of extra JVM options to the driver and the executors via `spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively. 
+  
+   * - `spark_env_vars`
+     - Map
+     - An object containing a set of optional, user-specified environment variable key-value pairs. Please note that key-value pair of the form (X,Y) will be exported as is (i.e., `export X='Y'`) while launching the driver and workers.  In order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending them to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all default databricks managed environmental variables are included as well.  Example Spark environment variables: `{"SPARK_WORKER_MEMORY": "28000m", "SPARK_LOCAL_DIRS": "/local_disk0"}` or `{"SPARK_DAEMON_JAVA_OPTS": "$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true"}`
+  
+   * - `spark_version`
+     - String
+     - The Spark version of the cluster, e.g. `3.3.x-scala2.11`. A list of available Spark versions can be retrieved by using the :method:clusters/sparkVersions API call. 
+  
+   * - `ssh_public_keys`
+     - Sequence
+     - SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified.
+  
+   * - `use_ml_runtime`
+     - Boolean
+     - This field can only be used with `kind`.  `effective_spark_version` is determined by `spark_version` (DBR release), this field `use_ml_runtime`, and whether `node_type_id` is gpu node or not. 
+  
+   * - `workload_type`
+     - Map
+     - See [_](#jobs.<name>.tasks.new_cluster.workload_type).
+  
+  
+### jobs.<name>.tasks.new_cluster.autoscale
+  
+**`Type: Map`**
+  
+Parameters needed in order to automatically scale clusters up and down based on load.
+Note: autoscaling works best with DB runtime versions 3.0 or later.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `max_workers`
+     - Integer
+     - The maximum number of workers to which the cluster can scale up when overloaded. Note that `max_workers` must be strictly greater than `min_workers`.
+  
+   * - `min_workers`
+     - Integer
+     - The minimum number of workers to which the cluster can scale down when underutilized. It is also the initial number of workers the cluster will have after creation.
+  
+  
+### jobs.<name>.tasks.new_cluster.aws_attributes
+  
+**`Type: Map`**
+  
+Attributes related to clusters running on Amazon Web Services.
+If not specified at cluster creation, a set of default values will be used.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `availability`
+     - String
+     - Availability type used for all subsequent nodes past the `first_on_demand` ones.  Note: If `first_on_demand` is zero, this availability type will be used for the entire cluster. 
+  
+   * - `ebs_volume_count`
+     - Integer
+     - The number of volumes launched for each instance. Users can choose up to 10 volumes. This feature is only enabled for supported node types. Legacy node types cannot specify custom EBS volumes. For node types with no instance store, at least one EBS volume needs to be specified; otherwise, cluster creation will fail.  These EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc. Instance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc.  If EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for scratch storage because heterogenously sized scratch devices can lead to inefficient disk utilization. If no EBS volumes are attached, Databricks will configure Spark to use instance store volumes.  Please note that if EBS volumes are specified, then the Spark configuration `spark.local.dir` will be overridden.
+  
+   * - `ebs_volume_iops`
+     - Integer
+     - If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used.
+  
+   * - `ebs_volume_size`
+     - Integer
+     - The size of each EBS volume (in GiB) launched for each instance. For general purpose SSD, this value must be within the range 100 - 4096. For throughput optimized HDD, this value must be within the range 500 - 4096.
+  
+   * - `ebs_volume_throughput`
+     - Integer
+     - If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used.
+  
+   * - `ebs_volume_type`
+     - String
+     - The type of EBS volumes that will be launched with this cluster.
+  
+   * - `first_on_demand`
+     - Integer
+     - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. If this value is greater than 0, the cluster driver node in particular will be placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster.
+  
+   * - `instance_profile_arn`
+     - String
+     - Nodes for this cluster will only be placed on AWS instances with this instance profile. If ommitted, nodes will be placed on instances without an IAM instance profile. The instance profile must have previously been added to the Databricks environment by an account administrator.  This feature may only be available to certain customer plans.  If this field is ommitted, we will pull in the default from the conf if it exists.
+  
+   * - `spot_bid_price_percent`
+     - Integer
+     - The bid price for AWS spot instances, as a percentage of the corresponding instance type's on-demand price. For example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot instance, then the bid price is half of the price of on-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice the price of on-demand `r3.xlarge` instances. If not specified, the default value is 100. When spot instances are requested for this cluster, only spot instances whose bid price percentage matches this field will be considered. Note that, for safety, we enforce this field to be no more than 10000.  The default value and documentation here should be kept consistent with CommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent.
+  
+   * - `zone_id`
+     - String
+     - Identifier for the availability zone/datacenter in which the cluster resides. This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. This is an optional field at cluster creation, and if not specified, a default zone will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. The list of available zones as well as the default value can be found by using the `List Zones` method.
+  
+  
+### jobs.<name>.tasks.new_cluster.azure_attributes
+  
+**`Type: Map`**
+  
+Attributes related to clusters running on Microsoft Azure.
+If not specified at cluster creation, a set of default values will be used.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `availability`
+     - String
+     - Availability type used for all subsequent nodes past the `first_on_demand` ones. Note: If `first_on_demand` is zero (which only happens on pool clusters), this availability type will be used for the entire cluster.
+  
+   * - `first_on_demand`
+     - Integer
+     - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster.
+  
+   * - `log_analytics_info`
+     - Map
+     - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#jobs.<name>.tasks.new_cluster.azure_attributes.log_analytics_info).
+  
+   * - `spot_bid_max_price`
+     - Any
+     - The max bid price to be used for Azure spot instances. The Max price for the bid cannot be higher than the on-demand price of the instance. If not specified, the default value is -1, which specifies that the instance cannot be evicted on the basis of price, and only on the basis of availability. Further, the value should > 0 or -1.
+  
+  
+### jobs.<name>.tasks.new_cluster.azure_attributes.log_analytics_info
+  
+**`Type: Map`**
+  
+Defines values necessary to configure and run Azure Log Analytics agent
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `log_analytics_primary_key`
+     - String
+     - <needs content added>
+  
+   * - `log_analytics_workspace_id`
+     - String
+     - <needs content added>
+  
+  
+### jobs.<name>.tasks.new_cluster.cluster_log_conf
+  
+**`Type: Map`**
+  
+The configuration for delivering spark logs to a long-term storage destination.
+Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified
+for one cluster. If the conf is given, the logs will be delivered to the destination every
+`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while
+the destination of executor logs is `$destination/$clusterId/executor`.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `dbfs`
+     - Map
+     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobs.<name>.tasks.new_cluster.cluster_log_conf.dbfs).
+  
+   * - `s3`
+     - Map
+     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobs.<name>.tasks.new_cluster.cluster_log_conf.s3).
+  
+  
+### jobs.<name>.tasks.new_cluster.cluster_log_conf.dbfs
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - dbfs destination, e.g. `dbfs:/my/path`
+  
+  
+### jobs.<name>.tasks.new_cluster.cluster_log_conf.s3
+  
+**`Type: Map`**
+  
+destination and either the region or endpoint need to be provided. e.g.
+`{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }`
+Cluster iam role is used to access s3, please make sure the cluster iam role in
+`instance_profile_arn` has permission to write data to the s3 destination.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `canned_acl`
+     - String
+     - (Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`. If `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on the destination bucket and prefix. The full list of possible canned acl can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl. Please also note that by default only the object owner gets full controls. If you are using cross account role for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to read the logs.
+  
+   * - `destination`
+     - String
+     - S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using cluster iam role, please make sure you set cluster iam role and the role has write access to the destination. Please also note that you cannot use AWS keys to deliver logs.
+  
+   * - `enable_encryption`
+     - Boolean
+     - (Optional) Flag to enable server side encryption, `false` by default.
+  
+   * - `encryption_type`
+     - String
+     - (Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when encryption is enabled and the default type is `sse-s3`.
+  
+   * - `endpoint`
+     - String
+     - S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
+  
+   * - `kms_key`
+     - String
+     - (Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`.
+  
+   * - `region`
+     - String
+     - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
+  
+  
+### jobs.<name>.tasks.new_cluster.docker_image
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `basic_auth`
+     - Map
+     - See [_](#jobs.<name>.tasks.new_cluster.docker_image.basic_auth).
+  
+   * - `url`
+     - String
+     - URL of the docker image.
+  
+  
+### jobs.<name>.tasks.new_cluster.docker_image.basic_auth
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `password`
+     - String
+     - Password of the user
+  
+   * - `username`
+     - String
+     - Name of the user
+  
+  
+### jobs.<name>.tasks.new_cluster.gcp_attributes
+  
+**`Type: Map`**
+  
+Attributes related to clusters running on Google Cloud Platform.
+If not specified at cluster creation, a set of default values will be used.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `availability`
+     - String
+     - This field determines whether the instance pool will contain preemptible VMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable.
+  
+   * - `boot_disk_size`
+     - Integer
+     - boot disk size in GB
+  
+   * - `google_service_account`
+     - String
+     - If provided, the cluster will impersonate the google service account when accessing gcloud services (like GCS). The google service account must have previously been added to the Databricks environment by an account administrator.
+  
+   * - `local_ssd_count`
+     - Integer
+     - If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type.
+  
+   * - `use_preemptible_executors`
+     - Boolean
+     - This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default). Note: Soon to be deprecated, use the availability field instead.
+  
+   * - `zone_id`
+     - String
+     - Identifier for the availability zone in which the cluster resides. This can be one of the following: - "HA" => High availability, spread nodes across availability zones for a Databricks deployment region [default] - "AUTO" => Databricks picks an availability zone to schedule the cluster on. - A GCP availability zone => Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones.
+  
+  
+### jobs.<name>.tasks.new_cluster.init_scripts
+  
+**`Type: Sequence`**
+  
+The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `<destination>/<cluster-ID>/init_scripts`.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `abfss`
+     - Map
+     - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>" } }. See [_](#jobs.<name>.tasks.new_cluster.init_scripts.abfss).
+  
+   * - `dbfs`
+     - Map
+     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobs.<name>.tasks.new_cluster.init_scripts.dbfs).
+  
+   * - `file`
+     - Map
+     - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#jobs.<name>.tasks.new_cluster.init_scripts.file).
+  
+   * - `gcs`
+     - Map
+     - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#jobs.<name>.tasks.new_cluster.init_scripts.gcs).
+  
+   * - `s3`
+     - Map
+     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobs.<name>.tasks.new_cluster.init_scripts.s3).
+  
+   * - `volumes`
+     - Map
+     - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#jobs.<name>.tasks.new_cluster.init_scripts.volumes).
+  
+   * - `workspace`
+     - Map
+     - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#jobs.<name>.tasks.new_cluster.init_scripts.workspace).
+  
+  
+### jobs.<name>.tasks.new_cluster.init_scripts.abfss
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "abfss" : { "destination" : "abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>" } }
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - abfss destination, e.g. `abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>`.
+  
+  
+### jobs.<name>.tasks.new_cluster.init_scripts.dbfs
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - dbfs destination, e.g. `dbfs:/my/path`
+  
+  
+### jobs.<name>.tasks.new_cluster.init_scripts.file
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "file" : { "destination" : "file:/my/local/file.sh" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - local file destination, e.g. `file:/my/local/file.sh`
+  
+  
+### jobs.<name>.tasks.new_cluster.init_scripts.gcs
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - GCS destination/URI, e.g. `gs://my-bucket/some-prefix`
+  
+  
+### jobs.<name>.tasks.new_cluster.init_scripts.s3
+  
+**`Type: Map`**
+  
+destination and either the region or endpoint need to be provided. e.g.
+`{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }`
+Cluster iam role is used to access s3, please make sure the cluster iam role in
+`instance_profile_arn` has permission to write data to the s3 destination.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `canned_acl`
+     - String
+     - (Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`. If `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on the destination bucket and prefix. The full list of possible canned acl can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl. Please also note that by default only the object owner gets full controls. If you are using cross account role for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to read the logs.
+  
+   * - `destination`
+     - String
+     - S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using cluster iam role, please make sure you set cluster iam role and the role has write access to the destination. Please also note that you cannot use AWS keys to deliver logs.
+  
+   * - `enable_encryption`
+     - Boolean
+     - (Optional) Flag to enable server side encryption, `false` by default.
+  
+   * - `encryption_type`
+     - String
+     - (Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when encryption is enabled and the default type is `sse-s3`.
+  
+   * - `endpoint`
+     - String
+     - S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
+  
+   * - `kms_key`
+     - String
+     - (Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`.
+  
+   * - `region`
+     - String
+     - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
+  
+  
+### jobs.<name>.tasks.new_cluster.init_scripts.volumes
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`
+  
+  
+### jobs.<name>.tasks.new_cluster.init_scripts.workspace
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`
+  
+  
+### jobs.<name>.tasks.new_cluster.workload_type
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `clients`
+     - Map
+     -  defined what type of clients can use the cluster. E.g. Notebooks, Jobs. See [_](#jobs.<name>.tasks.new_cluster.workload_type.clients).
+  
+  
+### jobs.<name>.tasks.new_cluster.workload_type.clients
+  
+**`Type: Map`**
+  
+ defined what type of clients can use the cluster. E.g. Notebooks, Jobs
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `jobs`
+     - Boolean
+     - With jobs set, the cluster can be used for jobs
+  
+   * - `notebooks`
+     - Boolean
+     - With notebooks set, this cluster can be used for notebooks
+  
+  
+### jobs.<name>.tasks.notebook_task
+  
+**`Type: Map`**
+  
+The task runs a notebook when the `notebook_task` field is present.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `base_parameters`
+     - Map
+     - Base parameters to be used for each run of this job. If the run is initiated by a call to :method:jobs/run Now with parameters specified, the two parameters maps are merged. If the same key is specified in `base_parameters` and in `run-now`, the value from `run-now` is used. Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.  If the notebook takes a parameter that is not specified in the job’s `base_parameters` or the `run-now` override parameters, the default value from the notebook is used.  Retrieve these parameters in a notebook using [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html#dbutils-widgets).  The JSON representation of this field cannot exceed 1MB.
+  
+   * - `notebook_path`
+     - String
+     - The path of the notebook to be run in the Databricks workspace or remote repository. For notebooks stored in the Databricks workspace, the path must be absolute and begin with a slash. For notebooks stored in a remote repository, the path must be relative. This field is required.
+  
+   * - `source`
+     - String
+     - Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository defined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise. * `WORKSPACE`: Notebook is located in Databricks workspace. * `GIT`: Notebook is located in cloud Git provider.
+  
+   * - `warehouse_id`
+     - String
+     - Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses.  Note that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail.
+  
+  
+### jobs.<name>.tasks.notification_settings
+  
+**`Type: Map`**
+  
+Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this task.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `alert_on_last_attempt`
+     - Boolean
+     - If true, do not send notifications to recipients specified in `on_start` for the retried runs and do not send notifications to recipients specified in `on_failure` until the last retry of the run.
+  
+   * - `no_alert_for_canceled_runs`
+     - Boolean
+     - If true, do not send notifications to recipients specified in `on_failure` if the run is canceled.
+  
+   * - `no_alert_for_skipped_runs`
+     - Boolean
+     - If true, do not send notifications to recipients specified in `on_failure` if the run is skipped.
+  
+  
+### jobs.<name>.tasks.pipeline_task
+  
+**`Type: Map`**
+  
+The task triggers a pipeline update when the `pipeline_task` field is present. Only pipelines configured to use triggered more are supported.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `full_refresh`
+     - Boolean
+     - If true, triggers a full refresh on the delta live table.
+  
+   * - `pipeline_id`
+     - String
+     - The full name of the pipeline task to execute.
+  
+  
+### jobs.<name>.tasks.python_wheel_task
+  
+**`Type: Map`**
+  
+The task runs a Python wheel when the `python_wheel_task` field is present.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `entry_point`
+     - String
+     - Named entry point to use, if it does not exist in the metadata of the package it executes the function from the package directly using `$packageName.$entryPoint()`
+  
+   * - `named_parameters`
+     - Map
+     - Command-line parameters passed to Python wheel task in the form of `["--name=task", "--data=dbfs:/path/to/data.json"]`. Leave it empty if `parameters` is not null.
+  
+   * - `package_name`
+     - String
+     - Name of the package to execute
+  
+   * - `parameters`
+     - Sequence
+     - Command-line parameters passed to Python wheel task. Leave it empty if `named_parameters` is not null.
+  
+  
+### jobs.<name>.tasks.run_job_task
+  
+**`Type: Map`**
+  
+The task triggers another job when the `run_job_task` field is present.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `dbt_commands`
+     - Sequence
+     - An array of commands to execute for jobs with the dbt task, for example `"dbt_commands": ["dbt deps", "dbt seed", "dbt deps", "dbt seed", "dbt run"]`
+  
+   * - `jar_params`
+     - Sequence
+     - A list of parameters for jobs with Spark JAR tasks, for example `"jar_params": ["john doe", "35"]`. The parameters are used to invoke the main function of the main class specified in the Spark JAR task. If not specified upon `run-now`, it defaults to an empty list. jar_params cannot be specified in conjunction with notebook_params. The JSON representation of this field (for example `{"jar_params":["john doe","35"]}`) cannot exceed 10,000 bytes.  Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.
+  
+   * - `job_id`
+     - Integer
+     - ID of the job to trigger.
+  
+   * - `job_parameters`
+     - Map
+     - Job-level parameters used to trigger the job.
+  
+   * - `notebook_params`
+     - Map
+     - A map from keys to values for jobs with notebook task, for example `"notebook_params": {"name": "john doe", "age": "35"}`. The map is passed to the notebook and is accessible through the [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html) function.  If not specified upon `run-now`, the triggered run uses the job’s base parameters.  notebook_params cannot be specified in conjunction with jar_params.  Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.  The JSON representation of this field (for example `{"notebook_params":{"name":"john doe","age":"35"}}`) cannot exceed 10,000 bytes.
+  
+   * - `pipeline_params`
+     - Map
+     - Controls whether the pipeline should perform a full refresh. See [_](#jobs.<name>.tasks.run_job_task.pipeline_params).
+  
+   * - `python_named_params`
+     - Map
+     - 
+  
+   * - `python_params`
+     - Sequence
+     - A list of parameters for jobs with Python tasks, for example `"python_params": ["john doe", "35"]`. The parameters are passed to Python file as command-line parameters. If specified upon `run-now`, it would overwrite the parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes.  Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.  Important  These parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.
+  
+   * - `spark_submit_params`
+     - Sequence
+     - A list of parameters for jobs with spark submit task, for example `"spark_submit_params": ["--class", "org.apache.spark.examples.SparkPi"]`. The parameters are passed to spark-submit script as command-line parameters. If specified upon `run-now`, it would overwrite the parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes.  Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs  Important  These parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.
+  
+   * - `sql_params`
+     - Map
+     - A map from keys to values for jobs with SQL task, for example `"sql_params": {"name": "john doe", "age": "35"}`. The SQL alert task does not support custom parameters.
+  
+  
+### jobs.<name>.tasks.run_job_task.pipeline_params
+  
+**`Type: Map`**
+  
+Controls whether the pipeline should perform a full refresh
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `full_refresh`
+     - Boolean
+     - If true, triggers a full refresh on the delta live table.
+  
+  
+### jobs.<name>.tasks.spark_jar_task
+  
+**`Type: Map`**
+  
+The task runs a JAR when the `spark_jar_task` field is present.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `jar_uri`
+     - String
+     - Deprecated since 04/2016. Provide a `jar` through the `libraries` field instead. For an example, see :method:jobs/create.
+  
+   * - `main_class_name`
+     - String
+     - The full name of the class containing the main method to be executed. This class must be contained in a JAR provided as a library.  The code must use `SparkContext.getOrCreate` to obtain a Spark context; otherwise, runs of the job fail.
+  
+   * - `parameters`
+     - Sequence
+     - Parameters passed to the main method.  Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.
+  
+   * - `run_as_repl`
+     - Boolean
+     - Deprecated. A value of `false` is no longer supported.
+  
+  
+### jobs.<name>.tasks.spark_python_task
+  
+**`Type: Map`**
+  
+The task runs a Python file when the `spark_python_task` field is present.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `parameters`
+     - Sequence
+     - Command line parameters passed to the Python file.  Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.
+  
+   * - `python_file`
+     - String
+     - The Python file to be executed. Cloud file URIs (such as dbfs:/, s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored in the Databricks workspace, the path must be absolute and begin with `/`. For files stored in a remote repository, the path must be relative. This field is required.
+  
+   * - `source`
+     - String
+     - Optional location type of the Python file. When set to `WORKSPACE` or not specified, the file will be retrieved from the local Databricks workspace or cloud location (if the `python_file` has a URI format). When set to `GIT`, the Python file will be retrieved from a Git repository defined in `git_source`.  * `WORKSPACE`: The Python file is located in a Databricks workspace or at a cloud filesystem URI. * `GIT`: The Python file is located in a remote Git repository.
+  
+  
+### jobs.<name>.tasks.spark_submit_task
+  
+**`Type: Map`**
+  
+(Legacy) The task runs the spark-submit script when the `spark_submit_task` field is present. This task can run only on new clusters and is not compatible with serverless compute.
+
+In the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark configurations.
+
+`master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you _cannot_ specify them in parameters.
+
+By default, the Spark submit job uses all available memory (excluding reserved memory for Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value to leave some room for off-heap usage.
+
+The `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `parameters`
+     - Sequence
+     - Command-line parameters passed to spark submit.  Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.
+  
+  
+### jobs.<name>.tasks.sql_task
+  
+**`Type: Map`**
+  
+The task runs a SQL query or file, or it refreshes a SQL alert or a legacy SQL dashboard when the `sql_task` field is present.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `alert`
+     - Map
+     - If alert, indicates that this job must refresh a SQL alert. See [_](#jobs.<name>.tasks.sql_task.alert).
+  
+   * - `dashboard`
+     - Map
+     - If dashboard, indicates that this job must refresh a SQL dashboard. See [_](#jobs.<name>.tasks.sql_task.dashboard).
+  
+   * - `file`
+     - Map
+     - If file, indicates that this job runs a SQL file in a remote Git repository. See [_](#jobs.<name>.tasks.sql_task.file).
+  
+   * - `parameters`
+     - Map
+     - Parameters to be used for each run of this job. The SQL alert task does not support custom parameters.
+  
+   * - `query`
+     - Map
+     - If query, indicates that this job must execute a SQL query. See [_](#jobs.<name>.tasks.sql_task.query).
+  
+   * - `warehouse_id`
+     - String
+     - The canonical identifier of the SQL warehouse. Recommended to use with serverless or pro SQL warehouses. Classic SQL warehouses are only supported for SQL alert, dashboard and query tasks and are limited to scheduled single-task jobs.
+  
+  
+### jobs.<name>.tasks.sql_task.alert
+  
+**`Type: Map`**
+  
+If alert, indicates that this job must refresh a SQL alert.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `alert_id`
+     - String
+     - The canonical identifier of the SQL alert.
+  
+   * - `pause_subscriptions`
+     - Boolean
+     - If true, the alert notifications are not sent to subscribers.
+  
+   * - `subscriptions`
+     - Sequence
+     - If specified, alert notifications are sent to subscribers. See [_](#jobs.<name>.tasks.sql_task.alert.subscriptions).
+  
+  
+### jobs.<name>.tasks.sql_task.alert.subscriptions
+  
+**`Type: Sequence`**
+  
+If specified, alert notifications are sent to subscribers.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination_id`
+     - String
+     - The canonical identifier of the destination to receive email notification. This parameter is mutually exclusive with user_name. You cannot set both destination_id and user_name for subscription notifications.
+  
+   * - `user_name`
+     - String
+     - The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications.
+  
+  
+### jobs.<name>.tasks.sql_task.dashboard
+  
+**`Type: Map`**
+  
+If dashboard, indicates that this job must refresh a SQL dashboard.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `custom_subject`
+     - String
+     - Subject of the email sent to subscribers of this task.
+  
+   * - `dashboard_id`
+     - String
+     - The canonical identifier of the SQL dashboard.
+  
+   * - `pause_subscriptions`
+     - Boolean
+     - If true, the dashboard snapshot is not taken, and emails are not sent to subscribers.
+  
+   * - `subscriptions`
+     - Sequence
+     - If specified, dashboard snapshots are sent to subscriptions. See [_](#jobs.<name>.tasks.sql_task.dashboard.subscriptions).
+  
+  
+### jobs.<name>.tasks.sql_task.dashboard.subscriptions
+  
+**`Type: Sequence`**
+  
+If specified, dashboard snapshots are sent to subscriptions.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination_id`
+     - String
+     - The canonical identifier of the destination to receive email notification. This parameter is mutually exclusive with user_name. You cannot set both destination_id and user_name for subscription notifications.
+  
+   * - `user_name`
+     - String
+     - The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications.
+  
+  
+### jobs.<name>.tasks.sql_task.file
+  
+**`Type: Map`**
+  
+If file, indicates that this job runs a SQL file in a remote Git repository.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `path`
+     - String
+     - Path of the SQL file. Must be relative if the source is a remote Git repository and absolute for workspace paths.
+  
+   * - `source`
+     - String
+     - Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved from the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository defined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.  * `WORKSPACE`: SQL file is located in Databricks workspace. * `GIT`: SQL file is located in cloud Git provider.
+  
+  
+### jobs.<name>.tasks.sql_task.query
+  
+**`Type: Map`**
+  
+If query, indicates that this job must execute a SQL query.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `query_id`
+     - String
+     - The canonical identifier of the SQL query.
+  
+  
+### jobs.<name>.tasks.webhook_notifications
+  
+**`Type: Map`**
+  
+A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `on_duration_warning_threshold_exceeded`
+     - Sequence
+     - An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property. See [_](#jobs.<name>.tasks.webhook_notifications.on_duration_warning_threshold_exceeded).
+  
+   * - `on_failure`
+     - Sequence
+     - An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property. See [_](#jobs.<name>.tasks.webhook_notifications.on_failure).
+  
+   * - `on_start`
+     - Sequence
+     - An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property. See [_](#jobs.<name>.tasks.webhook_notifications.on_start).
+  
+   * - `on_streaming_backlog_exceeded`
+     - Sequence
+     - An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes. A maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property. See [_](#jobs.<name>.tasks.webhook_notifications.on_streaming_backlog_exceeded).
+  
+   * - `on_success`
+     - Sequence
+     - An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property. See [_](#jobs.<name>.tasks.webhook_notifications.on_success).
+  
+  
+### jobs.<name>.tasks.webhook_notifications.on_duration_warning_threshold_exceeded
+  
+**`Type: Sequence`**
+  
+An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `id`
+     - String
+     - 
+  
+  
+### jobs.<name>.tasks.webhook_notifications.on_failure
+  
+**`Type: Sequence`**
+  
+An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `id`
+     - String
+     - 
+  
+  
+### jobs.<name>.tasks.webhook_notifications.on_start
+  
+**`Type: Sequence`**
+  
+An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `id`
+     - String
+     - 
+  
+  
+### jobs.<name>.tasks.webhook_notifications.on_streaming_backlog_exceeded
+  
+**`Type: Sequence`**
+  
+An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream.
+Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.
+Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.
+A maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `id`
+     - String
+     - 
+  
+  
+### jobs.<name>.tasks.webhook_notifications.on_success
+  
+**`Type: Sequence`**
+  
+An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `id`
+     - String
+     - 
+  
+  
+### jobs.<name>.trigger
+  
+**`Type: Map`**
+  
+A configuration to trigger a run when certain conditions are met. The default behavior is that the job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `file_arrival`
+     - Map
+     - File arrival trigger settings. See [_](#jobs.<name>.trigger.file_arrival).
+  
+   * - `pause_status`
+     - String
+     - Whether this trigger is paused or not.
+  
+   * - `periodic`
+     - Map
+     - Periodic trigger settings. See [_](#jobs.<name>.trigger.periodic).
+  
+   * - `table`
+     - Map
+     - Old table trigger settings name. Deprecated in favor of `table_update`. See [_](#jobs.<name>.trigger.table).
+  
+   * - `table_update`
+     - Map
+     - See [_](#jobs.<name>.trigger.table_update).
+  
+  
+### jobs.<name>.trigger.file_arrival
+  
+**`Type: Map`**
+  
+File arrival trigger settings.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `min_time_between_triggers_seconds`
+     - Integer
+     - If set, the trigger starts a run only after the specified amount of time passed since the last time the trigger fired. The minimum allowed value is 60 seconds
+  
+   * - `url`
+     - String
+     - URL to be monitored for file arrivals. The path must point to the root or a subpath of the external location.
+  
+   * - `wait_after_last_change_seconds`
+     - Integer
+     - If set, the trigger starts a run only after no file activity has occurred for the specified amount of time. This makes it possible to wait for a batch of incoming files to arrive before triggering a run. The minimum allowed value is 60 seconds.
+  
+  
+### jobs.<name>.trigger.periodic
+  
+**`Type: Map`**
+  
+Periodic trigger settings.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `interval`
+     - Integer
+     - The interval at which the trigger should run.
+  
+   * - `unit`
+     - String
+     - The unit of time for the interval.
+  
+  
+### jobs.<name>.trigger.table
+  
+**`Type: Map`**
+  
+Old table trigger settings name. Deprecated in favor of `table_update`.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `condition`
+     - String
+     - The table(s) condition based on which to trigger a job run.
+  
+   * - `min_time_between_triggers_seconds`
+     - Integer
+     - If set, the trigger starts a run only after the specified amount of time has passed since the last time the trigger fired. The minimum allowed value is 60 seconds.
+  
+   * - `table_names`
+     - Sequence
+     - A list of Delta tables to monitor for changes. The table name must be in the format `catalog_name.schema_name.table_name`.
+  
+   * - `wait_after_last_change_seconds`
+     - Integer
+     - If set, the trigger starts a run only after no table updates have occurred for the specified time and can be used to wait for a series of table updates before triggering a run. The minimum allowed value is 60 seconds.
+  
+  
+### jobs.<name>.trigger.table_update
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `condition`
+     - String
+     - The table(s) condition based on which to trigger a job run.
+  
+   * - `min_time_between_triggers_seconds`
+     - Integer
+     - If set, the trigger starts a run only after the specified amount of time has passed since the last time the trigger fired. The minimum allowed value is 60 seconds.
+  
+   * - `table_names`
+     - Sequence
+     - A list of Delta tables to monitor for changes. The table name must be in the format `catalog_name.schema_name.table_name`.
+  
+   * - `wait_after_last_change_seconds`
+     - Integer
+     - If set, the trigger starts a run only after no table updates have occurred for the specified time and can be used to wait for a series of table updates before triggering a run. The minimum allowed value is 60 seconds.
+  
+  
+### jobs.<name>.webhook_notifications
+  
+**`Type: Map`**
+  
+A collection of system notification IDs to notify when runs of this job begin or complete.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `on_duration_warning_threshold_exceeded`
+     - Sequence
+     - An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property. See [_](#jobs.<name>.webhook_notifications.on_duration_warning_threshold_exceeded).
+  
+   * - `on_failure`
+     - Sequence
+     - An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property. See [_](#jobs.<name>.webhook_notifications.on_failure).
+  
+   * - `on_start`
+     - Sequence
+     - An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property. See [_](#jobs.<name>.webhook_notifications.on_start).
+  
+   * - `on_streaming_backlog_exceeded`
+     - Sequence
+     - An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes. A maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property. See [_](#jobs.<name>.webhook_notifications.on_streaming_backlog_exceeded).
+  
+   * - `on_success`
+     - Sequence
+     - An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property. See [_](#jobs.<name>.webhook_notifications.on_success).
+  
+  
+### jobs.<name>.webhook_notifications.on_duration_warning_threshold_exceeded
+  
+**`Type: Sequence`**
+  
+An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `id`
+     - String
+     - 
+  
+  
+### jobs.<name>.webhook_notifications.on_failure
+  
+**`Type: Sequence`**
+  
+An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `id`
+     - String
+     - 
+  
+  
+### jobs.<name>.webhook_notifications.on_start
+  
+**`Type: Sequence`**
+  
+An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `id`
+     - String
+     - 
+  
+  
+### jobs.<name>.webhook_notifications.on_streaming_backlog_exceeded
+  
+**`Type: Sequence`**
+  
+An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream.
+Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.
+Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.
+A maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `id`
+     - String
+     - 
+  
+  
+### jobs.<name>.webhook_notifications.on_success
+  
+**`Type: Sequence`**
+  
+An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `id`
+     - String
+     - 
+  
+  
+## model_serving_endpoints
+  
+**`Type: Map`**
+  
+The model_serving_endpoint resource allows you to define [model serving endpoints](/api/workspace/servingendpoints/create). See [_](/machine-learning/model-serving/manage-serving-endpoints.md).
+  
+```yaml
+model_serving_endpoints:
+  <model_serving_endpoint-name>:
+    <model_serving_endpoint-field-name>: <model_serving_endpoint-field-value>
+```
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `ai_gateway`
+     - Map
+     - The AI Gateway configuration for the serving endpoint. NOTE: Only external model and provisioned throughput endpoints are currently supported. See [_](#model_serving_endpoints.<name>.ai_gateway).
+  
+   * - `config`
+     - Map
+     - The core config of the serving endpoint. See [_](#model_serving_endpoints.<name>.config).
+  
+   * - `name`
+     - String
+     - The name of the serving endpoint. This field is required and must be unique across a Databricks workspace. An endpoint name can consist of alphanumeric characters, dashes, and underscores.
+  
+   * - `permissions`
+     - Sequence
+     - See [_](#model_serving_endpoints.<name>.permissions).
+  
+   * - `rate_limits`
+     - Sequence
+     - Rate limits to be applied to the serving endpoint. NOTE: this field is deprecated, please use AI Gateway to manage rate limits. See [_](#model_serving_endpoints.<name>.rate_limits).
+  
+   * - `route_optimized`
+     - Boolean
+     - Enable route optimization for the serving endpoint.
+  
+   * - `tags`
+     - Sequence
+     - Tags to be attached to the serving endpoint and automatically propagated to billing logs. See [_](#model_serving_endpoints.<name>.tags).
+  
+  
+**Example**
+  
+The following example defines a <UC> model serving endpoint:
+
+```yaml
+resources:
+  model_serving_endpoints:
+    uc_model_serving_endpoint:
+      name: "uc-model-endpoint"
+      config:
+        served_entities:
+        - entity_name: "myCatalog.mySchema.my-ads-model"
+          entity_version: "10"
+          workload_size: "Small"
+          scale_to_zero_enabled: "true"
+        traffic_config:
+          routes:
+          - served_model_name: "my-ads-model-10"
+            traffic_percentage: "100"
+      tags:
+      - key: "team"
+        value: "data science"
+```
+  
+### model_serving_endpoints.<name>.ai_gateway
+  
+**`Type: Map`**
+  
+The AI Gateway configuration for the serving endpoint. NOTE: Only external model and provisioned throughput endpoints are currently supported.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `guardrails`
+     - Map
+     - Configuration for AI Guardrails to prevent unwanted data and unsafe data in requests and responses. See [_](#model_serving_endpoints.<name>.ai_gateway.guardrails).
+  
+   * - `inference_table_config`
+     - Map
+     - Configuration for payload logging using inference tables. Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality. See [_](#model_serving_endpoints.<name>.ai_gateway.inference_table_config).
+  
+   * - `rate_limits`
+     - Sequence
+     - Configuration for rate limits which can be set to limit endpoint traffic. See [_](#model_serving_endpoints.<name>.ai_gateway.rate_limits).
+  
+   * - `usage_tracking_config`
+     - Map
+     - Configuration to enable usage tracking using system tables. These tables allow you to monitor operational usage on endpoints and their associated costs. See [_](#model_serving_endpoints.<name>.ai_gateway.usage_tracking_config).
+  
+  
+### model_serving_endpoints.<name>.ai_gateway.guardrails
+  
+**`Type: Map`**
+  
+Configuration for AI Guardrails to prevent unwanted data and unsafe data in requests and responses.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `input`
+     - Map
+     - Configuration for input guardrail filters. See [_](#model_serving_endpoints.<name>.ai_gateway.guardrails.input).
+  
+   * - `output`
+     - Map
+     - Configuration for output guardrail filters. See [_](#model_serving_endpoints.<name>.ai_gateway.guardrails.output).
+  
+  
+### model_serving_endpoints.<name>.ai_gateway.guardrails.input
+  
+**`Type: Map`**
+  
+Configuration for input guardrail filters.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `invalid_keywords`
+     - Sequence
+     - List of invalid keywords. AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.
+  
+   * - `pii`
+     - Map
+     - Configuration for guardrail PII filter. See [_](#model_serving_endpoints.<name>.ai_gateway.guardrails.input.pii).
+  
+   * - `safety`
+     - Boolean
+     - Indicates whether the safety filter is enabled.
+  
+   * - `valid_topics`
+     - Sequence
+     - The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics.
+  
+  
+### model_serving_endpoints.<name>.ai_gateway.guardrails.input.pii
+  
+**`Type: Map`**
+  
+Configuration for guardrail PII filter.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `behavior`
+     - String
+     - Configuration for input guardrail filters.
+  
+  
+### model_serving_endpoints.<name>.ai_gateway.guardrails.output
+  
+**`Type: Map`**
+  
+Configuration for output guardrail filters.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `invalid_keywords`
+     - Sequence
+     - List of invalid keywords. AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.
+  
+   * - `pii`
+     - Map
+     - Configuration for guardrail PII filter. See [_](#model_serving_endpoints.<name>.ai_gateway.guardrails.output.pii).
+  
+   * - `safety`
+     - Boolean
+     - Indicates whether the safety filter is enabled.
+  
+   * - `valid_topics`
+     - Sequence
+     - The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics.
+  
+  
+### model_serving_endpoints.<name>.ai_gateway.guardrails.output.pii
+  
+**`Type: Map`**
+  
+Configuration for guardrail PII filter.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `behavior`
+     - String
+     - Configuration for input guardrail filters.
+  
+  
+### model_serving_endpoints.<name>.ai_gateway.inference_table_config
+  
+**`Type: Map`**
+  
+Configuration for payload logging using inference tables.
+Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `catalog_name`
+     - String
+     - The name of the catalog in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the catalog name.
+  
+   * - `enabled`
+     - Boolean
+     - Indicates whether the inference table is enabled.
+  
+   * - `schema_name`
+     - String
+     - The name of the schema in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the schema name.
+  
+   * - `table_name_prefix`
+     - String
+     - The prefix of the table in Unity Catalog. NOTE: On update, you have to disable inference table first in order to change the prefix name.
+  
+  
+### model_serving_endpoints.<name>.ai_gateway.rate_limits
+  
+**`Type: Sequence`**
+  
+Configuration for rate limits which can be set to limit endpoint traffic.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `calls`
+     - Integer
+     - Used to specify how many calls are allowed for a key within the renewal_period.
+  
+   * - `key`
+     - String
+     - Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.
+  
+   * - `renewal_period`
+     - String
+     - Renewal period field for a rate limit. Currently, only 'minute' is supported.
+  
+  
+### model_serving_endpoints.<name>.ai_gateway.usage_tracking_config
+  
+**`Type: Map`**
+  
+Configuration to enable usage tracking using system tables.
+These tables allow you to monitor operational usage on endpoints and their associated costs.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `enabled`
+     - Boolean
+     - Whether to enable usage tracking.
+  
+  
+### model_serving_endpoints.<name>.config
+  
+**`Type: Map`**
+  
+The core config of the serving endpoint.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `auto_capture_config`
+     - Map
+     - Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog. Note: this field is deprecated for creating new provisioned throughput endpoints, or updating existing provisioned throughput endpoints that never have inference table configured; in these cases please use AI Gateway to manage inference tables. See [_](#model_serving_endpoints.<name>.config.auto_capture_config).
+  
+   * - `served_entities`
+     - Sequence
+     - The list of served entities under the serving endpoint config. See [_](#model_serving_endpoints.<name>.config.served_entities).
+  
+   * - `served_models`
+     - Sequence
+     - (Deprecated, use served_entities instead) The list of served models under the serving endpoint config. See [_](#model_serving_endpoints.<name>.config.served_models).
+  
+   * - `traffic_config`
+     - Map
+     - The traffic configuration associated with the serving endpoint config. See [_](#model_serving_endpoints.<name>.config.traffic_config).
+  
+  
+### model_serving_endpoints.<name>.config.auto_capture_config
+  
+**`Type: Map`**
+  
+Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog.
+Note: this field is deprecated for creating new provisioned throughput endpoints,
+or updating existing provisioned throughput endpoints that never have inference table configured;
+in these cases please use AI Gateway to manage inference tables.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `catalog_name`
+     - String
+     - The name of the catalog in Unity Catalog. NOTE: On update, you cannot change the catalog name if the inference table is already enabled.
+  
+   * - `enabled`
+     - Boolean
+     - Indicates whether the inference table is enabled.
+  
+   * - `schema_name`
+     - String
+     - The name of the schema in Unity Catalog. NOTE: On update, you cannot change the schema name if the inference table is already enabled.
+  
+   * - `table_name_prefix`
+     - String
+     - The prefix of the table in Unity Catalog. NOTE: On update, you cannot change the prefix name if the inference table is already enabled.
+  
+  
+### model_serving_endpoints.<name>.config.served_entities
+  
+**`Type: Sequence`**
+  
+The list of served entities under the serving endpoint config.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `entity_name`
+     - String
+     - The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC), or a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of **catalog_name.schema_name.model_name**.
+  
+   * - `entity_version`
+     - String
+     - 
+  
+   * - `environment_vars`
+     - Map
+     - An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{"OPENAI_API_KEY": "{{secrets/my_scope/my_key}}", "DATABRICKS_TOKEN": "{{secrets/my_scope2/my_key2}}"}`
+  
+   * - `external_model`
+     - Map
+     - The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same. See [_](#model_serving_endpoints.<name>.config.served_entities.external_model).
+  
+   * - `instance_profile_arn`
+     - String
+     - ARN of the instance profile that the served entity uses to access AWS resources.
+  
+   * - `max_provisioned_throughput`
+     - Integer
+     - The maximum tokens per second that the endpoint can scale up to.
+  
+   * - `min_provisioned_throughput`
+     - Integer
+     - The minimum tokens per second that the endpoint can scale down to.
+  
+   * - `name`
+     - String
+     - The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.
+  
+   * - `scale_to_zero_enabled`
+     - Boolean
+     - Whether the compute resources for the served entity should scale down to zero.
+  
+   * - `workload_size`
+     - String
+     - The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.
+  
+   * - `workload_type`
+     - String
+     - The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).
+  
+  
+### model_serving_endpoints.<name>.config.served_entities.external_model
+  
+**`Type: Map`**
+  
+The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `ai21labs_config`
+     - Map
+     - AI21Labs Config. Only required if the provider is 'ai21labs'. See [_](#model_serving_endpoints.<name>.config.served_entities.external_model.ai21labs_config).
+  
+   * - `amazon_bedrock_config`
+     - Map
+     - Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'. See [_](#model_serving_endpoints.<name>.config.served_entities.external_model.amazon_bedrock_config).
+  
+   * - `anthropic_config`
+     - Map
+     - Anthropic Config. Only required if the provider is 'anthropic'. See [_](#model_serving_endpoints.<name>.config.served_entities.external_model.anthropic_config).
+  
+   * - `cohere_config`
+     - Map
+     - Cohere Config. Only required if the provider is 'cohere'. See [_](#model_serving_endpoints.<name>.config.served_entities.external_model.cohere_config).
+  
+   * - `databricks_model_serving_config`
+     - Map
+     - Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'. See [_](#model_serving_endpoints.<name>.config.served_entities.external_model.databricks_model_serving_config).
+  
+   * - `google_cloud_vertex_ai_config`
+     - Map
+     - Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'. See [_](#model_serving_endpoints.<name>.config.served_entities.external_model.google_cloud_vertex_ai_config).
+  
+   * - `name`
+     - String
+     - The name of the external model.
+  
+   * - `openai_config`
+     - Map
+     - OpenAI Config. Only required if the provider is 'openai'. See [_](#model_serving_endpoints.<name>.config.served_entities.external_model.openai_config).
+  
+   * - `palm_config`
+     - Map
+     - PaLM Config. Only required if the provider is 'palm'. See [_](#model_serving_endpoints.<name>.config.served_entities.external_model.palm_config).
+  
+   * - `provider`
+     - String
+     - The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic', 'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.
+  
+   * - `task`
+     - String
+     - The task type of the external model.
+  
+  
+### model_serving_endpoints.<name>.config.served_entities.external_model.ai21labs_config
+  
+**`Type: Map`**
+  
+AI21Labs Config. Only required if the provider is 'ai21labs'.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `ai21labs_api_key`
+     - String
+     - The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`.
+  
+   * - `ai21labs_api_key_plaintext`
+     - String
+     - An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`.
+  
+  
+### model_serving_endpoints.<name>.config.served_entities.external_model.amazon_bedrock_config
+  
+**`Type: Map`**
+  
+Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `aws_access_key_id`
+     - String
+     - The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id_plaintext`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.
+  
+   * - `aws_access_key_id_plaintext`
+     - String
+     - An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.
+  
+   * - `aws_region`
+     - String
+     - The AWS region to use. Bedrock has to be enabled there.
+  
+   * - `aws_secret_access_key`
+     - String
+     - The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.
+  
+   * - `aws_secret_access_key_plaintext`
+     - String
+     - An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.
+  
+   * - `bedrock_provider`
+     - String
+     - The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.
+  
+  
+### model_serving_endpoints.<name>.config.served_entities.external_model.anthropic_config
+  
+**`Type: Map`**
+  
+Anthropic Config. Only required if the provider is 'anthropic'.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `anthropic_api_key`
+     - String
+     - The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.
+  
+   * - `anthropic_api_key_plaintext`
+     - String
+     - The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.
+  
+  
+### model_serving_endpoints.<name>.config.served_entities.external_model.cohere_config
+  
+**`Type: Map`**
+  
+Cohere Config. Only required if the provider is 'cohere'.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `cohere_api_base`
+     - String
+     - This is an optional field to provide a customized base URL for the Cohere API. If left unspecified, the standard Cohere base URL is used.
+  
+   * - `cohere_api_key`
+     - String
+     - The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.
+  
+   * - `cohere_api_key_plaintext`
+     - String
+     - The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.
+  
+  
+### model_serving_endpoints.<name>.config.served_entities.external_model.databricks_model_serving_config
+  
+**`Type: Map`**
+  
+Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `databricks_api_token`
+     - String
+     - The Databricks secret key reference for a Databricks API token that corresponds to a user or service principal with Can Query access to the model serving endpoint pointed to by this external model. If you prefer to paste your API key directly, see `databricks_api_token_plaintext`. You must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.
+  
+   * - `databricks_api_token_plaintext`
+     - String
+     - The Databricks API token that corresponds to a user or service principal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `databricks_api_token`. You must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.
+  
+   * - `databricks_workspace_url`
+     - String
+     - The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model.
+  
+  
+### model_serving_endpoints.<name>.config.served_entities.external_model.google_cloud_vertex_ai_config
+  
+**`Type: Map`**
+  
+Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `private_key`
+     - String
+     - The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys]. If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`  [Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys
+  
+   * - `private_key_plaintext`
+     - String
+     - The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys]. If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`.  [Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys
+  
+   * - `project_id`
+     - String
+     - This is the Google Cloud project id that the service account is associated with.
+  
+   * - `region`
+     - String
+     - This is the region for the Google Cloud Vertex AI Service. See [supported regions] for more details. Some models are only available in specific regions.  [supported regions]: https://cloud.google.com/vertex-ai/docs/general/locations
+  
+  
+### model_serving_endpoints.<name>.config.served_entities.external_model.openai_config
+  
+**`Type: Map`**
+  
+OpenAI Config. Only required if the provider is 'openai'.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `microsoft_entra_client_id`
+     - String
+     - This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.
+  
+   * - `microsoft_entra_client_secret`
+     - String
+     - The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication. If you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`. You must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.
+  
+   * - `microsoft_entra_client_secret_plaintext`
+     - String
+     - The client secret used for Microsoft Entra ID authentication provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`. You must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.
+  
+   * - `microsoft_entra_tenant_id`
+     - String
+     - This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.
+  
+   * - `openai_api_base`
+     - String
+     - This is a field to provide a customized base URl for the OpenAI API. For Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service provided by Azure. For other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used.
+  
+   * - `openai_api_key`
+     - String
+     - The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`.
+  
+   * - `openai_api_key_plaintext`
+     - String
+     - The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`.
+  
+   * - `openai_api_type`
+     - String
+     - This is an optional field to specify the type of OpenAI API to use. For Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security access validation protocol. For access token validation, use azure. For authentication using Azure Active Directory (Azure AD) use, azuread.
+  
+   * - `openai_api_version`
+     - String
+     - This is an optional field to specify the OpenAI API version. For Azure OpenAI, this field is required, and is the version of the Azure OpenAI service to utilize, specified by a date.
+  
+   * - `openai_deployment_name`
+     - String
+     - This field is only required for Azure OpenAI and is the name of the deployment resource for the Azure OpenAI service.
+  
+   * - `openai_organization`
+     - String
+     - This is an optional field to specify the organization in OpenAI or Azure OpenAI.
+  
+  
+### model_serving_endpoints.<name>.config.served_entities.external_model.palm_config
+  
+**`Type: Map`**
+  
+PaLM Config. Only required if the provider is 'palm'.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `palm_api_key`
+     - String
+     - The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`.
+  
+   * - `palm_api_key_plaintext`
+     - String
+     - The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`.
+  
+  
+### model_serving_endpoints.<name>.config.served_models
+  
+**`Type: Sequence`**
+  
+(Deprecated, use served_entities instead) The list of served models under the serving endpoint config.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `environment_vars`
+     - Map
+     - An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{"OPENAI_API_KEY": "{{secrets/my_scope/my_key}}", "DATABRICKS_TOKEN": "{{secrets/my_scope2/my_key2}}"}`
+  
+   * - `instance_profile_arn`
+     - String
+     - ARN of the instance profile that the served entity uses to access AWS resources.
+  
+   * - `max_provisioned_throughput`
+     - Integer
+     - The maximum tokens per second that the endpoint can scale up to.
+  
+   * - `min_provisioned_throughput`
+     - Integer
+     - The minimum tokens per second that the endpoint can scale down to.
+  
+   * - `model_name`
+     - String
+     - 
+  
+   * - `model_version`
+     - String
+     - 
+  
+   * - `name`
+     - String
+     - The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.
+  
+   * - `scale_to_zero_enabled`
+     - Boolean
+     - Whether the compute resources for the served entity should scale down to zero.
+  
+   * - `workload_size`
+     - String
+     - The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.
+  
+   * - `workload_type`
+     - String
+     - The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).
+  
+  
+### model_serving_endpoints.<name>.config.traffic_config
+  
+**`Type: Map`**
+  
+The traffic configuration associated with the serving endpoint config.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `routes`
+     - Sequence
+     - The list of routes that define traffic to each served entity. See [_](#model_serving_endpoints.<name>.config.traffic_config.routes).
+  
+  
+### model_serving_endpoints.<name>.config.traffic_config.routes
+  
+**`Type: Sequence`**
+  
+The list of routes that define traffic to each served entity.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `served_model_name`
+     - String
+     - The name of the served model this route configures traffic for.
+  
+   * - `traffic_percentage`
+     - Integer
+     - The percentage of endpoint traffic to send to this route. It must be an integer between 0 and 100 inclusive.
+  
+  
+### model_serving_endpoints.<name>.permissions
+  
+**`Type: Sequence`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `group_name`
+     - String
+     - The name of the group that has the permission set in level.
+  
+   * - `level`
+     - String
+     - The allowed permission for user, group, service principal defined for this permission.
+  
+   * - `service_principal_name`
+     - String
+     - The name of the service principal that has the permission set in level.
+  
+   * - `user_name`
+     - String
+     - The name of the user that has the permission set in level.
+  
+  
+### model_serving_endpoints.<name>.rate_limits
+  
+**`Type: Sequence`**
+  
+Rate limits to be applied to the serving endpoint. NOTE: this field is deprecated, please use AI Gateway to manage rate limits.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `calls`
+     - Integer
+     - Used to specify how many calls are allowed for a key within the renewal_period.
+  
+   * - `key`
+     - String
+     - Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.
+  
+   * - `renewal_period`
+     - String
+     - Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported.
+  
+  
+### model_serving_endpoints.<name>.tags
+  
+**`Type: Sequence`**
+  
+Tags to be attached to the serving endpoint and automatically propagated to billing logs.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `key`
+     - String
+     - Key field for a serving endpoint tag.
+  
+   * - `value`
+     - String
+     - Optional value field for a serving endpoint tag.
+  
+  
+## models
+  
+**`Type: Map`**
+  
+The model resource allows you to define [legacy models](/api/workspace/modelregistry/createmodel) in bundles. Databricks recommends you use <UC> [registered models](#registered-model) instead.
+  
+```yaml
+models:
+  <model-name>:
+    <model-field-name>: <model-field-value>
+```
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `creation_timestamp`
+     - Integer
+     - Timestamp recorded when this `registered_model` was created.
+  
+   * - `description`
+     - String
+     - Description of this `registered_model`.
+  
+   * - `last_updated_timestamp`
+     - Integer
+     - Timestamp recorded when metadata for this `registered_model` was last updated.
+  
+   * - `latest_versions`
+     - Sequence
+     - Collection of latest model versions for each stage. Only contains models with current `READY` status. See [_](#models.<name>.latest_versions).
+  
+   * - `name`
+     - String
+     - Unique name for the model.
+  
+   * - `permissions`
+     - Sequence
+     - See [_](#models.<name>.permissions).
+  
+   * - `tags`
+     - Sequence
+     - Tags: Additional metadata key-value pairs for this `registered_model`. See [_](#models.<name>.tags).
+  
+   * - `user_id`
+     - String
+     - User that created this `registered_model`
+  
+  
+### models.<name>.latest_versions
+  
+**`Type: Sequence`**
+  
+Collection of latest model versions for each stage.
+Only contains models with current `READY` status.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `creation_timestamp`
+     - Integer
+     - Timestamp recorded when this `model_version` was created.
+  
+   * - `current_stage`
+     - String
+     - Current stage for this `model_version`.
+  
+   * - `description`
+     - String
+     - Description of this `model_version`.
+  
+   * - `last_updated_timestamp`
+     - Integer
+     - Timestamp recorded when metadata for this `model_version` was last updated.
+  
+   * - `name`
+     - String
+     - Unique name of the model
+  
+   * - `run_id`
+     - String
+     - MLflow run ID used when creating `model_version`, if `source` was generated by an experiment run stored in MLflow tracking server.
+  
+   * - `run_link`
+     - String
+     - Run Link: Direct link to the run that generated this version
+  
+   * - `source`
+     - String
+     - URI indicating the location of the source model artifacts, used when creating `model_version`
+  
+   * - `status`
+     - String
+     - Current status of `model_version`
+  
+   * - `status_message`
+     - String
+     - Details on current `status`, if it is pending or failed.
+  
+   * - `tags`
+     - Sequence
+     - Tags: Additional metadata key-value pairs for this `model_version`. See [_](#models.<name>.latest_versions.tags).
+  
+   * - `user_id`
+     - String
+     - User that created this `model_version`.
+  
+   * - `version`
+     - String
+     - Model's version number.
+  
+  
+### models.<name>.latest_versions.tags
+  
+**`Type: Sequence`**
+  
+Tags: Additional metadata key-value pairs for this `model_version`.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `key`
+     - String
+     - The tag key.
+  
+   * - `value`
+     - String
+     - The tag value.
+  
+  
+### models.<name>.permissions
+  
+**`Type: Sequence`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `group_name`
+     - String
+     - The name of the group that has the permission set in level.
+  
+   * - `level`
+     - String
+     - The allowed permission for user, group, service principal defined for this permission.
+  
+   * - `service_principal_name`
+     - String
+     - The name of the service principal that has the permission set in level.
+  
+   * - `user_name`
+     - String
+     - The name of the user that has the permission set in level.
+  
+  
+### models.<name>.tags
+  
+**`Type: Sequence`**
+  
+Tags: Additional metadata key-value pairs for this `registered_model`.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `key`
+     - String
+     - The tag key.
+  
+   * - `value`
+     - String
+     - The tag value.
+  
+  
+## pipelines
+  
+**`Type: Map`**
+  
+The pipeline resource allows you to create <DLT> [pipelines](/api/workspace/pipelines/create). For information about pipelines, see [_](/delta-live-tables/index.md). For a tutorial that uses the <DABS> template to create a pipeline, see [_](/dev-tools/bundles/pipelines-tutorial.md).
+  
+```yaml
+pipelines:
+  <pipeline-name>:
+    <pipeline-field-name>: <pipeline-field-value>
+```
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `budget_policy_id`
+     - String
+     - Budget policy of this pipeline.
+  
+   * - `catalog`
+     - String
+     - A catalog in Unity Catalog to publish data from this pipeline to. If `target` is specified, tables in this pipeline are published to a `target` schema inside `catalog` (for example, `catalog`.`target`.`table`). If `target` is not specified, no data is published to Unity Catalog.
+  
+   * - `channel`
+     - String
+     - DLT Release Channel that specifies which version to use.
+  
+   * - `clusters`
+     - Sequence
+     - Cluster settings for this pipeline deployment. See [_](#pipelines.<name>.clusters).
+  
+   * - `configuration`
+     - Map
+     - String-String configuration for this pipeline execution.
+  
+   * - `continuous`
+     - Boolean
+     - Whether the pipeline is continuous or triggered. This replaces `trigger`.
+  
+   * - `deployment`
+     - Map
+     - Deployment type of this pipeline. See [_](#pipelines.<name>.deployment).
+  
+   * - `development`
+     - Boolean
+     - Whether the pipeline is in Development mode. Defaults to false.
+  
+   * - `edition`
+     - String
+     - Pipeline product edition.
+  
+   * - `filters`
+     - Map
+     - Filters on which Pipeline packages to include in the deployed graph. See [_](#pipelines.<name>.filters).
+  
+   * - `gateway_definition`
+     - Map
+     - The definition of a gateway pipeline to support change data capture. See [_](#pipelines.<name>.gateway_definition).
+  
+   * - `id`
+     - String
+     - Unique identifier for this pipeline.
+  
+   * - `ingestion_definition`
+     - Map
+     - The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings. See [_](#pipelines.<name>.ingestion_definition).
+  
+   * - `libraries`
+     - Sequence
+     - Libraries or code needed by this deployment. See [_](#pipelines.<name>.libraries).
+  
+   * - `name`
+     - String
+     - Friendly identifier for this pipeline.
+  
+   * - `notifications`
+     - Sequence
+     - List of notification settings for this pipeline. See [_](#pipelines.<name>.notifications).
+  
+   * - `permissions`
+     - Sequence
+     - See [_](#pipelines.<name>.permissions).
+  
+   * - `photon`
+     - Boolean
+     - Whether Photon is enabled for this pipeline.
+  
+   * - `restart_window`
+     - Map
+     - Restart window of this pipeline. See [_](#pipelines.<name>.restart_window).
+  
+   * - `schema`
+     - String
+     - The default schema (database) where tables are read from or published to. The presence of this field implies that the pipeline is in direct publishing mode.
+  
+   * - `serverless`
+     - Boolean
+     - Whether serverless compute is enabled for this pipeline.
+  
+   * - `storage`
+     - String
+     - DBFS root directory for storing checkpoints and tables.
+  
+   * - `target`
+     - String
+     - Target schema (database) to add tables in this pipeline to. If not specified, no data is published to the Hive metastore or Unity Catalog. To publish to Unity Catalog, also specify `catalog`.
+  
+   * - `trigger`
+     - Map
+     - Which pipeline trigger to use. Deprecated: Use `continuous` instead. See [_](#pipelines.<name>.trigger).
+  
+  
+**Example**
+  
+The following example defines a pipeline with the resource key `hello-pipeline`:
+
+```yaml
+resources:
+  pipelines:
+    hello-pipeline:
+      name: hello-pipeline
+      clusters:
+        - label: default
+          num_workers: 1
+      development: true
+      continuous: false
+      channel: CURRENT
+      edition: CORE
+      photon: false
+      libraries:
+        - notebook:
+            path: ./pipeline.py
+```
+  
+### pipelines.<name>.clusters
+  
+**`Type: Sequence`**
+  
+Cluster settings for this pipeline deployment.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `apply_policy_default_values`
+     - Boolean
+     - Note: This field won't be persisted. Only API users will check this field.
+  
+   * - `autoscale`
+     - Map
+     - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#pipelines.<name>.clusters.autoscale).
+  
+   * - `aws_attributes`
+     - Map
+     - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#pipelines.<name>.clusters.aws_attributes).
+  
+   * - `azure_attributes`
+     - Map
+     - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#pipelines.<name>.clusters.azure_attributes).
+  
+   * - `cluster_log_conf`
+     - Map
+     - The configuration for delivering spark logs to a long-term storage destination. Only dbfs destinations are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. . See [_](#pipelines.<name>.clusters.cluster_log_conf).
+  
+   * - `custom_tags`
+     - Map
+     - Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS instances and EBS volumes) with these tags in addition to `default_tags`. Notes:  - Currently, Databricks allows at most 45 custom tags  - Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags
+  
+   * - `driver_instance_pool_id`
+     - String
+     - The optional ID of the instance pool for the driver of the cluster belongs. The pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not assigned.
+  
+   * - `driver_node_type_id`
+     - String
+     - The node type of the Spark driver. Note that this field is optional; if unset, the driver node type will be set as the same value as `node_type_id` defined above.
+  
+   * - `enable_local_disk_encryption`
+     - Boolean
+     - Whether to enable local disk encryption for the cluster.
+  
+   * - `gcp_attributes`
+     - Map
+     - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#pipelines.<name>.clusters.gcp_attributes).
+  
+   * - `init_scripts`
+     - Sequence
+     - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `<destination>/<cluster-ID>/init_scripts`. See [_](#pipelines.<name>.clusters.init_scripts).
+  
+   * - `instance_pool_id`
+     - String
+     - The optional ID of the instance pool to which the cluster belongs.
+  
+   * - `label`
+     - String
+     - A label for the cluster specification, either `default` to configure the default cluster, or `maintenance` to configure the maintenance cluster. This field is optional. The default value is `default`.
+  
+   * - `node_type_id`
+     - String
+     - This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or compute intensive workloads. A list of available node types can be retrieved by using the :method:clusters/listNodeTypes API call. 
+  
+   * - `num_workers`
+     - Integer
+     - Number of worker nodes that this cluster should have. A cluster has one Spark Driver and `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.  Note: When reading the properties of a cluster, this field reflects the desired number of workers rather than the actual current number of workers. For instance, if a cluster is resized from 5 to 10 workers, this field will immediately be updated to reflect the target size of 10 workers, whereas the workers listed in `spark_info` will gradually increase from 5 to 10 as the new nodes are provisioned.
+  
+   * - `policy_id`
+     - String
+     - The ID of the cluster policy used to create the cluster if applicable.
+  
+   * - `spark_conf`
+     - Map
+     - An object containing a set of optional, user-specified Spark configuration key-value pairs. See :method:clusters/create for more details. 
+  
+   * - `spark_env_vars`
+     - Map
+     - An object containing a set of optional, user-specified environment variable key-value pairs. Please note that key-value pair of the form (X,Y) will be exported as is (i.e., `export X='Y'`) while launching the driver and workers.  In order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending them to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all default databricks managed environmental variables are included as well.  Example Spark environment variables: `{"SPARK_WORKER_MEMORY": "28000m", "SPARK_LOCAL_DIRS": "/local_disk0"}` or `{"SPARK_DAEMON_JAVA_OPTS": "$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true"}`
+  
+   * - `ssh_public_keys`
+     - Sequence
+     - SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified.
+  
+  
+### pipelines.<name>.clusters.autoscale
+  
+**`Type: Map`**
+  
+Parameters needed in order to automatically scale clusters up and down based on load.
+Note: autoscaling works best with DB runtime versions 3.0 or later.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `max_workers`
+     - Integer
+     - The maximum number of workers to which the cluster can scale up when overloaded. `max_workers` must be strictly greater than `min_workers`.
+  
+   * - `min_workers`
+     - Integer
+     - The minimum number of workers the cluster can scale down to when underutilized. It is also the initial number of workers the cluster will have after creation.
+  
+   * - `mode`
+     - String
+     - Databricks Enhanced Autoscaling optimizes cluster utilization by automatically allocating cluster resources based on workload volume, with minimal impact to the data processing latency of your pipelines. Enhanced Autoscaling is available for `updates` clusters only. The legacy autoscaling feature is used for `maintenance` clusters. 
+  
+  
+### pipelines.<name>.clusters.aws_attributes
+  
+**`Type: Map`**
+  
+Attributes related to clusters running on Amazon Web Services.
+If not specified at cluster creation, a set of default values will be used.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `availability`
+     - String
+     - Availability type used for all subsequent nodes past the `first_on_demand` ones.  Note: If `first_on_demand` is zero, this availability type will be used for the entire cluster. 
+  
+   * - `ebs_volume_count`
+     - Integer
+     - The number of volumes launched for each instance. Users can choose up to 10 volumes. This feature is only enabled for supported node types. Legacy node types cannot specify custom EBS volumes. For node types with no instance store, at least one EBS volume needs to be specified; otherwise, cluster creation will fail.  These EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc. Instance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc.  If EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for scratch storage because heterogenously sized scratch devices can lead to inefficient disk utilization. If no EBS volumes are attached, Databricks will configure Spark to use instance store volumes.  Please note that if EBS volumes are specified, then the Spark configuration `spark.local.dir` will be overridden.
+  
+   * - `ebs_volume_iops`
+     - Integer
+     - If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used.
+  
+   * - `ebs_volume_size`
+     - Integer
+     - The size of each EBS volume (in GiB) launched for each instance. For general purpose SSD, this value must be within the range 100 - 4096. For throughput optimized HDD, this value must be within the range 500 - 4096.
+  
+   * - `ebs_volume_throughput`
+     - Integer
+     - If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used.
+  
+   * - `ebs_volume_type`
+     - String
+     - The type of EBS volumes that will be launched with this cluster.
+  
+   * - `first_on_demand`
+     - Integer
+     - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. If this value is greater than 0, the cluster driver node in particular will be placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster.
+  
+   * - `instance_profile_arn`
+     - String
+     - Nodes for this cluster will only be placed on AWS instances with this instance profile. If ommitted, nodes will be placed on instances without an IAM instance profile. The instance profile must have previously been added to the Databricks environment by an account administrator.  This feature may only be available to certain customer plans.  If this field is ommitted, we will pull in the default from the conf if it exists.
+  
+   * - `spot_bid_price_percent`
+     - Integer
+     - The bid price for AWS spot instances, as a percentage of the corresponding instance type's on-demand price. For example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot instance, then the bid price is half of the price of on-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice the price of on-demand `r3.xlarge` instances. If not specified, the default value is 100. When spot instances are requested for this cluster, only spot instances whose bid price percentage matches this field will be considered. Note that, for safety, we enforce this field to be no more than 10000.  The default value and documentation here should be kept consistent with CommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent.
+  
+   * - `zone_id`
+     - String
+     - Identifier for the availability zone/datacenter in which the cluster resides. This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. This is an optional field at cluster creation, and if not specified, a default zone will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. The list of available zones as well as the default value can be found by using the `List Zones` method.
+  
+  
+### pipelines.<name>.clusters.azure_attributes
+  
+**`Type: Map`**
+  
+Attributes related to clusters running on Microsoft Azure.
+If not specified at cluster creation, a set of default values will be used.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `availability`
+     - String
+     - Availability type used for all subsequent nodes past the `first_on_demand` ones. Note: If `first_on_demand` is zero (which only happens on pool clusters), this availability type will be used for the entire cluster.
+  
+   * - `first_on_demand`
+     - Integer
+     - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster.
+  
+   * - `log_analytics_info`
+     - Map
+     - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#pipelines.<name>.clusters.azure_attributes.log_analytics_info).
+  
+   * - `spot_bid_max_price`
+     - Any
+     - The max bid price to be used for Azure spot instances. The Max price for the bid cannot be higher than the on-demand price of the instance. If not specified, the default value is -1, which specifies that the instance cannot be evicted on the basis of price, and only on the basis of availability. Further, the value should > 0 or -1.
+  
+  
+### pipelines.<name>.clusters.azure_attributes.log_analytics_info
+  
+**`Type: Map`**
+  
+Defines values necessary to configure and run Azure Log Analytics agent
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `log_analytics_primary_key`
+     - String
+     - <needs content added>
+  
+   * - `log_analytics_workspace_id`
+     - String
+     - <needs content added>
+  
+  
+### pipelines.<name>.clusters.cluster_log_conf
+  
+**`Type: Map`**
+  
+The configuration for delivering spark logs to a long-term storage destination.
+Only dbfs destinations are supported. Only one destination can be specified
+for one cluster. If the conf is given, the logs will be delivered to the destination every
+`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while
+the destination of executor logs is `$destination/$clusterId/executor`.
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `dbfs`
+     - Map
+     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#pipelines.<name>.clusters.cluster_log_conf.dbfs).
+  
+   * - `s3`
+     - Map
+     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#pipelines.<name>.clusters.cluster_log_conf.s3).
+  
+  
+### pipelines.<name>.clusters.cluster_log_conf.dbfs
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - dbfs destination, e.g. `dbfs:/my/path`
+  
+  
+### pipelines.<name>.clusters.cluster_log_conf.s3
+  
+**`Type: Map`**
+  
+destination and either the region or endpoint need to be provided. e.g.
+`{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }`
+Cluster iam role is used to access s3, please make sure the cluster iam role in
+`instance_profile_arn` has permission to write data to the s3 destination.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `canned_acl`
+     - String
+     - (Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`. If `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on the destination bucket and prefix. The full list of possible canned acl can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl. Please also note that by default only the object owner gets full controls. If you are using cross account role for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to read the logs.
+  
+   * - `destination`
+     - String
+     - S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using cluster iam role, please make sure you set cluster iam role and the role has write access to the destination. Please also note that you cannot use AWS keys to deliver logs.
+  
+   * - `enable_encryption`
+     - Boolean
+     - (Optional) Flag to enable server side encryption, `false` by default.
+  
+   * - `encryption_type`
+     - String
+     - (Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when encryption is enabled and the default type is `sse-s3`.
+  
+   * - `endpoint`
+     - String
+     - S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
+  
+   * - `kms_key`
+     - String
+     - (Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`.
+  
+   * - `region`
+     - String
+     - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
+  
+  
+### pipelines.<name>.clusters.gcp_attributes
+  
+**`Type: Map`**
+  
+Attributes related to clusters running on Google Cloud Platform.
+If not specified at cluster creation, a set of default values will be used.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `availability`
+     - String
+     - This field determines whether the instance pool will contain preemptible VMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable.
+  
+   * - `boot_disk_size`
+     - Integer
+     - boot disk size in GB
+  
+   * - `google_service_account`
+     - String
+     - If provided, the cluster will impersonate the google service account when accessing gcloud services (like GCS). The google service account must have previously been added to the Databricks environment by an account administrator.
+  
+   * - `local_ssd_count`
+     - Integer
+     - If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type.
+  
+   * - `use_preemptible_executors`
+     - Boolean
+     - This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default). Note: Soon to be deprecated, use the availability field instead.
+  
+   * - `zone_id`
+     - String
+     - Identifier for the availability zone in which the cluster resides. This can be one of the following: - "HA" => High availability, spread nodes across availability zones for a Databricks deployment region [default] - "AUTO" => Databricks picks an availability zone to schedule the cluster on. - A GCP availability zone => Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones.
+  
+  
+### pipelines.<name>.clusters.init_scripts
+  
+**`Type: Sequence`**
+  
+The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `<destination>/<cluster-ID>/init_scripts`.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `abfss`
+     - Map
+     - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>" } }. See [_](#pipelines.<name>.clusters.init_scripts.abfss).
+  
+   * - `dbfs`
+     - Map
+     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#pipelines.<name>.clusters.init_scripts.dbfs).
+  
+   * - `file`
+     - Map
+     - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#pipelines.<name>.clusters.init_scripts.file).
+  
+   * - `gcs`
+     - Map
+     - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#pipelines.<name>.clusters.init_scripts.gcs).
+  
+   * - `s3`
+     - Map
+     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#pipelines.<name>.clusters.init_scripts.s3).
+  
+   * - `volumes`
+     - Map
+     - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#pipelines.<name>.clusters.init_scripts.volumes).
+  
+   * - `workspace`
+     - Map
+     - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#pipelines.<name>.clusters.init_scripts.workspace).
+  
+  
+### pipelines.<name>.clusters.init_scripts.abfss
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "abfss" : { "destination" : "abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>" } }
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - abfss destination, e.g. `abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>`.
+  
+  
+### pipelines.<name>.clusters.init_scripts.dbfs
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - dbfs destination, e.g. `dbfs:/my/path`
+  
+  
+### pipelines.<name>.clusters.init_scripts.file
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "file" : { "destination" : "file:/my/local/file.sh" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - local file destination, e.g. `file:/my/local/file.sh`
+  
+  
+### pipelines.<name>.clusters.init_scripts.gcs
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - GCS destination/URI, e.g. `gs://my-bucket/some-prefix`
+  
+  
+### pipelines.<name>.clusters.init_scripts.s3
+  
+**`Type: Map`**
+  
+destination and either the region or endpoint need to be provided. e.g.
+`{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }`
+Cluster iam role is used to access s3, please make sure the cluster iam role in
+`instance_profile_arn` has permission to write data to the s3 destination.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `canned_acl`
+     - String
+     - (Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`. If `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on the destination bucket and prefix. The full list of possible canned acl can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl. Please also note that by default only the object owner gets full controls. If you are using cross account role for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to read the logs.
+  
+   * - `destination`
+     - String
+     - S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using cluster iam role, please make sure you set cluster iam role and the role has write access to the destination. Please also note that you cannot use AWS keys to deliver logs.
+  
+   * - `enable_encryption`
+     - Boolean
+     - (Optional) Flag to enable server side encryption, `false` by default.
+  
+   * - `encryption_type`
+     - String
+     - (Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when encryption is enabled and the default type is `sse-s3`.
+  
+   * - `endpoint`
+     - String
+     - S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
+  
+   * - `kms_key`
+     - String
+     - (Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`.
+  
+   * - `region`
+     - String
+     - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
+  
+  
+### pipelines.<name>.clusters.init_scripts.volumes
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`
+  
+  
+### pipelines.<name>.clusters.init_scripts.workspace
+  
+**`Type: Map`**
+  
+destination needs to be provided. e.g.
+`{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination`
+     - String
+     - workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`
+  
+  
+### pipelines.<name>.deployment
+  
+**`Type: Map`**
+  
+Deployment type of this pipeline.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `kind`
+     - String
+     - The deployment method that manages the pipeline.
+  
+   * - `metadata_file_path`
+     - String
+     - The path to the file containing metadata about the deployment.
+  
+  
+### pipelines.<name>.filters
+  
+**`Type: Map`**
+  
+Filters on which Pipeline packages to include in the deployed graph.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `exclude`
+     - Sequence
+     - Paths to exclude.
+  
+   * - `include`
+     - Sequence
+     - Paths to include.
+  
+  
+### pipelines.<name>.gateway_definition
+  
+**`Type: Map`**
+  
+The definition of a gateway pipeline to support change data capture.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `connection_id`
+     - String
+     - [Deprecated, use connection_name instead] Immutable. The Unity Catalog connection that this gateway pipeline uses to communicate with the source.
+  
+   * - `connection_name`
+     - String
+     - Immutable. The Unity Catalog connection that this gateway pipeline uses to communicate with the source.
+  
+   * - `gateway_storage_catalog`
+     - String
+     - Required, Immutable. The name of the catalog for the gateway pipeline's storage location.
+  
+   * - `gateway_storage_name`
+     - String
+     - Optional. The Unity Catalog-compatible name for the gateway storage location. This is the destination to use for the data that is extracted by the gateway. Delta Live Tables system will automatically create the storage location under the catalog and schema. 
+  
+   * - `gateway_storage_schema`
+     - String
+     - Required, Immutable. The name of the schema for the gateway pipelines's storage location.
+  
+  
+### pipelines.<name>.ingestion_definition
+  
+**`Type: Map`**
+  
+The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `connection_name`
+     - String
+     - Immutable. The Unity Catalog connection that this ingestion pipeline uses to communicate with the source. This is used with connectors for applications like Salesforce, Workday, and so on.
+  
+   * - `ingestion_gateway_id`
+     - String
+     - Immutable. Identifier for the gateway that is used by this ingestion pipeline to communicate with the source database. This is used with connectors to databases like SQL Server.
+  
+   * - `objects`
+     - Sequence
+     - Required. Settings specifying tables to replicate and the destination for the replicated tables. See [_](#pipelines.<name>.ingestion_definition.objects).
+  
+   * - `table_configuration`
+     - Map
+     - Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline. See [_](#pipelines.<name>.ingestion_definition.table_configuration).
+  
+  
+### pipelines.<name>.ingestion_definition.objects
+  
+**`Type: Sequence`**
+  
+Required. Settings specifying tables to replicate and the destination for the replicated tables.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `report`
+     - Map
+     - Select a specific source report. See [_](#pipelines.<name>.ingestion_definition.objects.report).
+  
+   * - `schema`
+     - Map
+     - Select all tables from a specific source schema. See [_](#pipelines.<name>.ingestion_definition.objects.schema).
+  
+   * - `table`
+     - Map
+     - Select a specific source table. See [_](#pipelines.<name>.ingestion_definition.objects.table).
+  
+  
+### pipelines.<name>.ingestion_definition.objects.report
+  
+**`Type: Map`**
+  
+Select a specific source report.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination_catalog`
+     - String
+     - Required. Destination catalog to store table.
+  
+   * - `destination_schema`
+     - String
+     - Required. Destination schema to store table.
+  
+   * - `destination_table`
+     - String
+     - Required. Destination table name. The pipeline fails if a table with that name already exists.
+  
+   * - `source_url`
+     - String
+     - Required. Report URL in the source system.
+  
+   * - `table_configuration`
+     - Map
+     - Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object. See [_](#pipelines.<name>.ingestion_definition.objects.report.table_configuration).
+  
+  
+### pipelines.<name>.ingestion_definition.objects.report.table_configuration
+  
+**`Type: Map`**
+  
+Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `primary_keys`
+     - Sequence
+     - The primary key of the table used to apply changes.
+  
+   * - `salesforce_include_formula_fields`
+     - Boolean
+     - If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector
+  
+   * - `scd_type`
+     - String
+     - The SCD type to use to ingest the table.
+  
+   * - `sequence_by`
+     - Sequence
+     - The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order.
+  
+  
+### pipelines.<name>.ingestion_definition.objects.schema
+  
+**`Type: Map`**
+  
+Select all tables from a specific source schema.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination_catalog`
+     - String
+     - Required. Destination catalog to store tables.
+  
+   * - `destination_schema`
+     - String
+     - Required. Destination schema to store tables in. Tables with the same name as the source tables are created in this destination schema. The pipeline fails If a table with the same name already exists.
+  
+   * - `source_catalog`
+     - String
+     - The source catalog name. Might be optional depending on the type of source.
+  
+   * - `source_schema`
+     - String
+     - Required. Schema name in the source database.
+  
+   * - `table_configuration`
+     - Map
+     - Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object. See [_](#pipelines.<name>.ingestion_definition.objects.schema.table_configuration).
+  
+  
+### pipelines.<name>.ingestion_definition.objects.schema.table_configuration
+  
+**`Type: Map`**
+  
+Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `primary_keys`
+     - Sequence
+     - The primary key of the table used to apply changes.
+  
+   * - `salesforce_include_formula_fields`
+     - Boolean
+     - If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector
+  
+   * - `scd_type`
+     - String
+     - The SCD type to use to ingest the table.
+  
+   * - `sequence_by`
+     - Sequence
+     - The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order.
+  
+  
+### pipelines.<name>.ingestion_definition.objects.table
+  
+**`Type: Map`**
+  
+Select a specific source table.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `destination_catalog`
+     - String
+     - Required. Destination catalog to store table.
+  
+   * - `destination_schema`
+     - String
+     - Required. Destination schema to store table.
+  
+   * - `destination_table`
+     - String
+     - Optional. Destination table name. The pipeline fails if a table with that name already exists. If not set, the source table name is used.
+  
+   * - `source_catalog`
+     - String
+     - Source catalog name. Might be optional depending on the type of source.
+  
+   * - `source_schema`
+     - String
+     - Schema name in the source database. Might be optional depending on the type of source.
+  
+   * - `source_table`
+     - String
+     - Required. Table name in the source database.
+  
+   * - `table_configuration`
+     - Map
+     - Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec. See [_](#pipelines.<name>.ingestion_definition.objects.table.table_configuration).
+  
+  
+### pipelines.<name>.ingestion_definition.objects.table.table_configuration
+  
+**`Type: Map`**
+  
+Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `primary_keys`
+     - Sequence
+     - The primary key of the table used to apply changes.
+  
+   * - `salesforce_include_formula_fields`
+     - Boolean
+     - If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector
+  
+   * - `scd_type`
+     - String
+     - The SCD type to use to ingest the table.
+  
+   * - `sequence_by`
+     - Sequence
+     - The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order.
+  
+  
+### pipelines.<name>.ingestion_definition.table_configuration
+  
+**`Type: Map`**
+  
+Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `primary_keys`
+     - Sequence
+     - The primary key of the table used to apply changes.
+  
+   * - `salesforce_include_formula_fields`
+     - Boolean
+     - If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector
+  
+   * - `scd_type`
+     - String
+     - The SCD type to use to ingest the table.
+  
+   * - `sequence_by`
+     - Sequence
+     - The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order.
+  
+  
+### pipelines.<name>.libraries
+  
+**`Type: Sequence`**
+  
+Libraries or code needed by this deployment.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `file`
+     - Map
+     - The path to a file that defines a pipeline and is stored in the Databricks Repos. . See [_](#pipelines.<name>.libraries.file).
+  
+   * - `jar`
+     - String
+     - URI of the jar to be installed. Currently only DBFS is supported. 
+  
+   * - `maven`
+     - Map
+     - Specification of a maven library to be installed. . See [_](#pipelines.<name>.libraries.maven).
+  
+   * - `notebook`
+     - Map
+     - The path to a notebook that defines a pipeline and is stored in the Databricks workspace. . See [_](#pipelines.<name>.libraries.notebook).
+  
+   * - `whl`
+     - String
+     - URI of the whl to be installed.
+  
+  
+### pipelines.<name>.libraries.file
+  
+**`Type: Map`**
+  
+The path to a file that defines a pipeline and is stored in the Databricks Repos.
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `path`
+     - String
+     - The absolute path of the file.
+  
+  
+### pipelines.<name>.libraries.maven
+  
+**`Type: Map`**
+  
+Specification of a maven library to be installed.
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `coordinates`
+     - String
+     - Gradle-style maven coordinates. For example: "org.jsoup:jsoup:1.7.2".
+  
+   * - `exclusions`
+     - Sequence
+     - List of dependences to exclude. For example: `["slf4j:slf4j", "*:hadoop-client"]`.  Maven dependency exclusions: https://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html.
+  
+   * - `repo`
+     - String
+     - Maven repo to install the Maven package from. If omitted, both Maven Central Repository and Spark Packages are searched.
+  
+  
+### pipelines.<name>.libraries.notebook
+  
+**`Type: Map`**
+  
+The path to a notebook that defines a pipeline and is stored in the Databricks workspace.
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `path`
+     - String
+     - The absolute path of the notebook.
+  
+  
+### pipelines.<name>.notifications
+  
+**`Type: Sequence`**
+  
+List of notification settings for this pipeline.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `alerts`
+     - Sequence
+     - A list of alerts that trigger the sending of notifications to the configured destinations. The supported alerts are:  * `on-update-success`: A pipeline update completes successfully. * `on-update-failure`: Each time a pipeline update fails. * `on-update-fatal-failure`: A pipeline update fails with a non-retryable (fatal) error. * `on-flow-failure`: A single data flow fails. 
+  
+   * - `email_recipients`
+     - Sequence
+     - A list of email addresses notified when a configured alert is triggered. 
+  
+  
+### pipelines.<name>.permissions
+  
+**`Type: Sequence`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `group_name`
+     - String
+     - The name of the group that has the permission set in level.
+  
+   * - `level`
+     - String
+     - The allowed permission for user, group, service principal defined for this permission.
+  
+   * - `service_principal_name`
+     - String
+     - The name of the service principal that has the permission set in level.
+  
+   * - `user_name`
+     - String
+     - The name of the user that has the permission set in level.
+  
+  
+### pipelines.<name>.restart_window
+  
+**`Type: Map`**
+  
+Restart window of this pipeline.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `days_of_week`
+     - Sequence
+     - Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour). If not specified all days of the week will be used.
+  
+   * - `start_hour`
+     - Integer
+     - An integer between 0 and 23 denoting the start hour for the restart window in the 24-hour day. Continuous pipeline restart is triggered only within a five-hour window starting at this hour.
+  
+   * - `time_zone_id`
+     - String
+     - Time zone id of restart window. See https://docs.databricks.com/sql/language-manual/sql-ref-syntax-aux-conf-mgmt-set-timezone.html for details. If not specified, UTC will be used.
+  
+  
+### pipelines.<name>.restart_window.days_of_week
+  
+**`Type: Sequence`**
+  
+Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour).
+If not specified all days of the week will be used.
+  
+  
+### pipelines.<name>.trigger
+  
+**`Type: Map`**
+  
+Which pipeline trigger to use. Deprecated: Use `continuous` instead.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `cron`
+     - Map
+     - See [_](#pipelines.<name>.trigger.cron).
+  
+   * - `manual`
+     - Map
+     - 
+  
+  
+### pipelines.<name>.trigger.cron
+  
+**`Type: Map`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `quartz_cron_schedule`
+     - String
+     - 
+  
+   * - `timezone_id`
+     - String
+     - 
+  
+  
+### pipelines.<name>.trigger.manual
+  
+**`Type: Map`**
+  
+
+  
+  
+## quality_monitors
+  
+**`Type: Map`**
+  
+The quality_monitor resource allows you to define a <UC> [table monitor](/api/workspace/qualitymonitors/create). For information about monitors, see [_](/machine-learning/model-serving/monitor-diagnose-endpoints.md).
+  
+```yaml
+quality_monitors:
+  <quality_monitor-name>:
+    <quality_monitor-field-name>: <quality_monitor-field-value>
+```
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `assets_dir`
+     - String
+     - The directory to store monitoring assets (e.g. dashboard, metric tables).
+  
+   * - `baseline_table_name`
+     - String
+     - Name of the baseline table from which drift metrics are computed from. Columns in the monitored table should also be present in the baseline table. 
+  
+   * - `custom_metrics`
+     - Sequence
+     - Custom metrics to compute on the monitored table. These can be aggregate metrics, derived metrics (from already computed aggregate metrics), or drift metrics (comparing metrics across time windows). . See [_](#quality_monitors.<name>.custom_metrics).
+  
+   * - `data_classification_config`
+     - Map
+     - The data classification config for the monitor. See [_](#quality_monitors.<name>.data_classification_config).
+  
+   * - `inference_log`
+     - Map
+     - Configuration for monitoring inference logs. See [_](#quality_monitors.<name>.inference_log).
+  
+   * - `notifications`
+     - Map
+     - The notification settings for the monitor. See [_](#quality_monitors.<name>.notifications).
+  
+   * - `output_schema_name`
+     - String
+     - Schema where output metric tables are created.
+  
+   * - `schedule`
+     - Map
+     - The schedule for automatically updating and refreshing metric tables. See [_](#quality_monitors.<name>.schedule).
+  
+   * - `skip_builtin_dashboard`
+     - Boolean
+     - Whether to skip creating a default dashboard summarizing data quality metrics.
+  
+   * - `slicing_exprs`
+     - Sequence
+     - List of column expressions to slice data with for targeted analysis. The data is grouped by each expression independently, resulting in a separate slice for each predicate and its complements. For high-cardinality columns, only the top 100 unique values by frequency will generate slices. 
+  
+   * - `snapshot`
+     - Map
+     - Configuration for monitoring snapshot tables.
+  
+   * - `table_name`
+     - String
+     - 
+  
+   * - `time_series`
+     - Map
+     - Configuration for monitoring time series tables. See [_](#quality_monitors.<name>.time_series).
+  
+   * - `warehouse_id`
+     - String
+     - Optional argument to specify the warehouse for dashboard creation. If not specified, the first running warehouse will be used. 
+  
+  
+**Example**
+  
+The following example defines a quality monitor:
+
+```yaml
+resources:
+  quality_monitors:
+    my_quality_monitor:
+      table_name: dev.mlops_schema.predictions
+      output_schema_name: ${bundle.target}.mlops_schema
+      assets_dir: /Users/${workspace.current_user.userName}/databricks_lakehouse_monitoring
+      inference_log:
+        granularities: [1 day]
+        model_id_col: model_id
+        prediction_col: prediction
+        label_col: price
+        problem_type: PROBLEM_TYPE_REGRESSION
+        timestamp_col: timestamp
+      schedule:
+        quartz_cron_expression: 0 0 8 * * ? # Run Every day at 8am
+        timezone_id: UTC
+```
+  
+### quality_monitors.<name>.custom_metrics
+  
+**`Type: Sequence`**
+  
+Custom metrics to compute on the monitored table. These can be aggregate metrics, derived
+metrics (from already computed aggregate metrics), or drift metrics (comparing metrics across time
+windows).
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `definition`
+     - String
+     - Jinja template for a SQL expression that specifies how to compute the metric. See [create metric definition](https://docs.databricks.com/en/lakehouse-monitoring/custom-metrics.html#create-definition).
+  
+   * - `input_columns`
+     - Sequence
+     - A list of column names in the input table the metric should be computed for. Can use ``":table"`` to indicate that the metric needs information from multiple columns. 
+  
+   * - `name`
+     - String
+     - Name of the metric in the output tables.
+  
+   * - `output_data_type`
+     - String
+     - The output type of the custom metric.
+  
+   * - `type`
+     - String
+     - Can only be one of ``"CUSTOM_METRIC_TYPE_AGGREGATE"``, ``"CUSTOM_METRIC_TYPE_DERIVED"``, or ``"CUSTOM_METRIC_TYPE_DRIFT"``. The ``"CUSTOM_METRIC_TYPE_AGGREGATE"`` and ``"CUSTOM_METRIC_TYPE_DERIVED"`` metrics are computed on a single table, whereas the ``"CUSTOM_METRIC_TYPE_DRIFT"`` compare metrics across baseline and input table, or across the two consecutive time windows. - CUSTOM_METRIC_TYPE_AGGREGATE: only depend on the existing columns in your table - CUSTOM_METRIC_TYPE_DERIVED: depend on previously computed aggregate metrics - CUSTOM_METRIC_TYPE_DRIFT:  depend on previously computed aggregate or derived metrics 
+  
+  
+### quality_monitors.<name>.data_classification_config
+  
+**`Type: Map`**
+  
+The data classification config for the monitor.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `enabled`
+     - Boolean
+     - Whether data classification is enabled.
+  
+  
+### quality_monitors.<name>.inference_log
+  
+**`Type: Map`**
+  
+Configuration for monitoring inference logs.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `granularities`
+     - Sequence
+     - Granularities for aggregating data into time windows based on their timestamp. Currently the following static granularities are supported: {``"5 minutes"``, ``"30 minutes"``, ``"1 hour"``, ``"1 day"``, ``"<n> week(s)"``, ``"1 month"``, ``"1 year"``}. 
+  
+   * - `label_col`
+     - String
+     - Optional column that contains the ground truth for the prediction.
+  
+   * - `model_id_col`
+     - String
+     - Column that contains the id of the model generating the predictions. Metrics will be computed per model id by default, and also across all model ids. 
+  
+   * - `prediction_col`
+     - String
+     - Column that contains the output/prediction from the model.
+  
+   * - `prediction_proba_col`
+     - String
+     - Optional column that contains the prediction probabilities for each class in a classification problem type. The values in this column should be a map, mapping each class label to the prediction probability for a given sample. The map should be of PySpark MapType(). 
+  
+   * - `problem_type`
+     - String
+     - Problem type the model aims to solve. Determines the type of model-quality metrics that will be computed.
+  
+   * - `timestamp_col`
+     - String
+     - Column that contains the timestamps of requests. The column must be one of the following: - A ``TimestampType`` column - A column whose values can be converted to timestamps through the pyspark   ``to_timestamp`` [function](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html). 
+  
+  
+### quality_monitors.<name>.notifications
+  
+**`Type: Map`**
+  
+The notification settings for the monitor.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `on_failure`
+     - Map
+     - Who to send notifications to on monitor failure. See [_](#quality_monitors.<name>.notifications.on_failure).
+  
+   * - `on_new_classification_tag_detected`
+     - Map
+     - Who to send notifications to when new data classification tags are detected. See [_](#quality_monitors.<name>.notifications.on_new_classification_tag_detected).
+  
+  
+### quality_monitors.<name>.notifications.on_failure
+  
+**`Type: Map`**
+  
+Who to send notifications to on monitor failure.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `email_addresses`
+     - Sequence
+     - The list of email addresses to send the notification to. A maximum of 5 email addresses is supported.
+  
+  
+### quality_monitors.<name>.notifications.on_new_classification_tag_detected
+  
+**`Type: Map`**
+  
+Who to send notifications to when new data classification tags are detected.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `email_addresses`
+     - Sequence
+     - The list of email addresses to send the notification to. A maximum of 5 email addresses is supported.
+  
+  
+### quality_monitors.<name>.schedule
+  
+**`Type: Map`**
+  
+The schedule for automatically updating and refreshing metric tables.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `pause_status`
+     - String
+     - Read only field that indicates whether a schedule is paused or not.
+  
+   * - `quartz_cron_expression`
+     - String
+     - The expression that determines when to run the monitor. See [examples](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html). 
+  
+   * - `timezone_id`
+     - String
+     - The timezone id (e.g., ``"PST"``) in which to evaluate the quartz expression. 
+  
+  
+### quality_monitors.<name>.snapshot
+  
+**`Type: Map`**
+  
+Configuration for monitoring snapshot tables.
+  
+  
+### quality_monitors.<name>.time_series
+  
+**`Type: Map`**
+  
+Configuration for monitoring time series tables.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `granularities`
+     - Sequence
+     - Granularities for aggregating data into time windows based on their timestamp. Currently the following static granularities are supported: {``"5 minutes"``, ``"30 minutes"``, ``"1 hour"``, ``"1 day"``, ``"<n> week(s)"``, ``"1 month"``, ``"1 year"``}. 
+  
+   * - `timestamp_col`
+     - String
+     - Column that contains the timestamps of requests. The column must be one of the following: - A ``TimestampType`` column - A column whose values can be converted to timestamps through the pyspark   ``to_timestamp`` [function](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html). 
+  
+  
+## registered_models
+  
+**`Type: Map`**
+  
+The registered model resource allows you to define models in <UC>. For information about <UC> [registered models](/api/workspace/registeredmodels/create), see [_](/machine-learning/manage-model-lifecycle/index.md).
+  
+```yaml
+registered_models:
+  <registered_model-name>:
+    <registered_model-field-name>: <registered_model-field-value>
+```
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `catalog_name`
+     - String
+     - The name of the catalog where the schema and the registered model reside
+  
+   * - `comment`
+     - String
+     - The comment attached to the registered model
+  
+   * - `grants`
+     - Sequence
+     - See [_](#registered_models.<name>.grants).
+  
+   * - `name`
+     - String
+     - The name of the registered model
+  
+   * - `schema_name`
+     - String
+     - The name of the schema where the registered model resides
+  
+   * - `storage_location`
+     - String
+     - The storage location on the cloud under which model version data files are stored
+  
+  
+**Example**
+  
+The following example defines a registered model in <UC>:
+
+```yaml
+resources:
+  registered_models:
+      model:
+        name: my_model
+        catalog_name: ${bundle.target}
+        schema_name: mlops_schema
+        comment: Registered model in Unity Catalog for ${bundle.target} deployment target
+        grants:
+          - privileges:
+              - EXECUTE
+            principal: account users
+```
+  
+### registered_models.<name>.grants
+  
+**`Type: Sequence`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `principal`
+     - String
+     - The name of the principal that will be granted privileges
+  
+   * - `privileges`
+     - Sequence
+     - The privileges to grant to the specified entity
+  
+  
+## schemas
+  
+**`Type: Map`**
+  
+The schema resource type allows you to define <UC> [schemas](/api/workspace/schemas/create) for tables and other assets in your workflows and pipelines created as part of a bundle. A schema, different from other resource types, has the following limitations:
+
+- The owner of a schema resource is always the deployment user, and cannot be changed. If `run_as` is specified in the bundle, it will be ignored by operations on the schema.
+- Only fields supported by the corresponding [Schemas object create API](/api/workspace/schemas/create) are available for the schema resource. For example, `enable_predictive_optimization` is not supported as it is only available on the [update API](/api/workspace/schemas/update).
+  
+```yaml
+schemas:
+  <schema-name>:
+    <schema-field-name>: <schema-field-value>
+```
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `catalog_name`
+     - String
+     - Name of parent catalog.
+  
+   * - `comment`
+     - String
+     - User-provided free-form text description.
+  
+   * - `grants`
+     - Sequence
+     - See [_](#schemas.<name>.grants).
+  
+   * - `name`
+     - String
+     - Name of schema, relative to parent catalog.
+  
+   * - `properties`
+     - Map
+     - 
+  
+   * - `storage_root`
+     - String
+     - Storage root URL for managed tables within schema.
+  
+  
+**Example**
+  
+The following example defines a pipeline with the resource key `my_pipeline` that creates a <UC> schema with the key `my_schema` as the target:
+
+```yaml
+resources:
+  pipelines:
+    my_pipeline:
+      name: test-pipeline-{{.unique_id}}
+      libraries:
+        - notebook:
+            path: ./nb.sql
+      development: true
+      catalog: main
+      target: ${resources.schemas.my_schema.id}
+
+  schemas:
+    my_schema:
+      name: test-schema-{{.unique_id}}
+      catalog_name: main
+      comment: This schema was created by DABs.
+```
+
+A top-level grants mapping is not supported by <DABS>, so if you want to set grants for a schema, define the grants for the schema within the `schemas` mapping. For more information about grants, see [_](/data-governance/unity-catalog/manage-privileges/index.md#grant).
+
+The following example defines a <UC> schema with grants:
+
+```yaml
+resources:
+  schemas:
+    my_schema:
+      name: test-schema
+      grants:
+        - principal: users
+          privileges:
+            - CAN_MANAGE
+        - principal: my_team
+          privileges:
+            - CAN_READ
+      catalog_name: main
+  ```
+  
+### schemas.<name>.grants
+  
+**`Type: Sequence`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `principal`
+     - String
+     - The name of the principal that will be granted privileges
+  
+   * - `privileges`
+     - Sequence
+     - The privileges to grant to the specified entity
+  
+  
+## volumes
+  
+**`Type: Map`**
+  
+The volume resource type allows you to define and create <UC> [volumes](/api/workspace/volumes/create) as part of a bundle. When deploying a bundle with a volume defined, note that:
+
+- A volume cannot be referenced in the `artifact_path` for the bundle until it exists in the workspace. Hence, if you want to use <DABS> to create the volume, you must first define the volume in the bundle, deploy it to create the volume, then reference it in the `artifact_path` in subsequent deployments.
+
+- Volumes in the bundle are not prepended with the `dev_${workspace.current_user.short_name}` prefix when the deployment target has `mode: development` configured. However, you can manually configure this prefix. See [_](/dev-tools/bundles/deployment-modes.md#custom-presets).
+  
+```yaml
+volumes:
+  <volume-name>:
+    <volume-field-name>: <volume-field-value>
+```
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `catalog_name`
+     - String
+     - The name of the catalog where the schema and the volume are
+  
+   * - `comment`
+     - String
+     - The comment attached to the volume
+  
+   * - `grants`
+     - Sequence
+     - See [_](#volumes.<name>.grants).
+  
+   * - `name`
+     - String
+     - The name of the volume
+  
+   * - `schema_name`
+     - String
+     - The name of the schema where the volume is
+  
+   * - `storage_location`
+     - String
+     - The storage location on the cloud
+  
+   * - `volume_type`
+     - String
+     - 
+  
+  
+**Example**
+  
+The following example creates a <UC> volume with the key `my_volume`:
+
+```yaml
+resources:
+  volumes:
+    my_volume:
+      catalog_name: main
+      name: my_volume
+      schema_name: my_schema
+```
+
+For an example bundle that runs a job that writes to a file in <UC> volume, see the [bundle-examples GitHub repository](https://github.com/databricks/bundle-examples/tree/main/knowledge_base/write_from_job_to_volume).
+  
+### volumes.<name>.grants
+  
+**`Type: Sequence`**
+  
+
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `principal`
+     - String
+     - The name of the principal that will be granted privileges
+  
+   * - `privileges`
+     - Sequence
+     - The privileges to grant to the specified entity
+  
\ No newline at end of file
diff --git a/bundle/docsgen/refs.go b/bundle/docsgen/refs.go
new file mode 100644
index 000000000..ca45e6ab2
--- /dev/null
+++ b/bundle/docsgen/refs.go
@@ -0,0 +1,97 @@
+package main
+
+import (
+	"log"
+	"strings"
+
+	"github.com/databricks/cli/libs/jsonschema"
+)
+
+func isReferenceType(v *jsonschema.Schema, refs map[string]*jsonschema.Schema, ownFields map[string]bool) bool {
+	if v.Type != "object" && v.Type != "array" {
+		return false
+	}
+	if len(v.Properties) > 0 {
+		return true
+	}
+	if v.Items != nil {
+		items := resolveRefs(v.Items, refs)
+		if items != nil && items.Type == "object" {
+			return true
+		}
+	}
+	props := resolveAdditionalProperties(v)
+	if !isInOwnFields(props, ownFields) {
+		return false
+	}
+	if props != nil {
+		propsResolved := resolveRefs(props, refs)
+		return propsResolved.Type == "object"
+	}
+
+	return false
+}
+
+func isInOwnFields(node *jsonschema.Schema, ownFields map[string]bool) bool {
+	if node != nil && node.Reference != nil {
+		return ownFields[getRefType(node)]
+	}
+	return true
+}
+
+func resolveAdditionalProperties(v *jsonschema.Schema) *jsonschema.Schema {
+	if v.AdditionalProperties == nil {
+		return nil
+	}
+	additionalProps, ok := v.AdditionalProperties.(*jsonschema.Schema)
+	if !ok {
+		return nil
+	}
+	return additionalProps
+}
+
+func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *jsonschema.Schema {
+	if s == nil {
+		return nil
+	}
+
+	node := s
+	description := s.Description
+	markdownDescription := s.MarkdownDescription
+	examples := s.Examples
+
+	for node.Reference != nil {
+		ref := getRefType(node)
+		newNode, ok := schemas[ref]
+		if !ok {
+			log.Printf("schema %s not found", ref)
+			break
+		}
+
+		if description == "" {
+			description = newNode.Description
+		}
+		if markdownDescription == "" {
+			markdownDescription = newNode.MarkdownDescription
+		}
+		if len(examples) == 0 {
+			examples = newNode.Examples
+		}
+
+		node = newNode
+	}
+
+	newNode := *node
+	newNode.Description = description
+	newNode.MarkdownDescription = markdownDescription
+	newNode.Examples = examples
+
+	return &newNode
+}
+
+func getRefType(node *jsonschema.Schema) string {
+	if node.Reference == nil {
+		return ""
+	}
+	return strings.TrimPrefix(*node.Reference, "#/$defs/")
+}
diff --git a/bundle/docsgen/renderer.go b/bundle/docsgen/renderer.go
new file mode 100644
index 000000000..5f6c77258
--- /dev/null
+++ b/bundle/docsgen/renderer.go
@@ -0,0 +1,51 @@
+package main
+
+import (
+	"fmt"
+	"runtime"
+	"strings"
+)
+
+type markdownRenderer struct {
+	nodes []string
+}
+
+func newMardownRenderer() *markdownRenderer {
+	return &markdownRenderer{}
+}
+
+func (m *markdownRenderer) add(s string) *markdownRenderer {
+	m.nodes = append(m.nodes, s)
+	return m
+}
+
+func (m *markdownRenderer) PlainText(s string) *markdownRenderer {
+	return m.add(s)
+}
+
+func (m *markdownRenderer) LF() *markdownRenderer {
+	return m.add("  ")
+}
+
+func (m *markdownRenderer) H2(s string) *markdownRenderer {
+	return m.add("## " + s)
+}
+
+func (m *markdownRenderer) H3(s string) *markdownRenderer {
+	return m.add("### " + s)
+}
+
+func (m *markdownRenderer) CodeBlocks(lang, s string) *markdownRenderer {
+	return m.add(fmt.Sprintf("```%s%s%s%s```", lang, lineFeed(), s, lineFeed()))
+}
+
+func (m *markdownRenderer) String() string {
+	return strings.Join(m.nodes, lineFeed())
+}
+
+func lineFeed() string {
+	if runtime.GOOS == "windows" {
+		return "\r\n"
+	}
+	return "\n"
+}
diff --git a/bundle/docsgen/templates/reference.md b/bundle/docsgen/templates/reference.md
new file mode 100644
index 000000000..a17d53315
--- /dev/null
+++ b/bundle/docsgen/templates/reference.md
@@ -0,0 +1,10 @@
+<!-- DO NOT EDIT. This file is autogenerated with https://github.com/databricks/cli -->
+---
+description: Configuration reference for databricks.yml
+---
+
+# Configuration reference
+
+This article provides reference for keys supported by <DABS> configuration (YAML). See [_](/dev-tools/bundles/index.md).
+
+For complete bundle examples, see [_](/dev-tools/bundles/resource-examples.md) and the [bundle-examples GitHub repository](https://github.com/databricks/bundle-examples).
diff --git a/bundle/docsgen/templates/resources.md b/bundle/docsgen/templates/resources.md
new file mode 100644
index 000000000..fccfac47d
--- /dev/null
+++ b/bundle/docsgen/templates/resources.md
@@ -0,0 +1,70 @@
+<!-- DO NOT EDIT. This file is autogenerated with https://github.com/databricks/cli -->
+---
+description: Learn about resources supported by Databricks Asset Bundles and how to configure them.
+---
+
+# <DABS> resources
+
+<DABS> allows you to specify information about the <Databricks> resources used by the bundle in the `resources` mapping in the bundle configuration. See [resources mapping](/dev-tools/bundles/settings.md#resources) and [resources key reference](/dev-tools/bundles/reference.md#resources).
+
+This article outlines supported resource types for bundles and provides details and an example for each supported type. For additional examples, see [_](/dev-tools/bundles/resource-examples.md).
+
+## <a id="resource-types"></a> Supported resources
+
+The following table lists supported resource types for bundles. Some resources can be created by defining them in a bundle and deploying the bundle, and some resources only support referencing an existing resource to include in the bundle.
+
+Resources are defined using the corresponding [Databricks REST API](/api/workspace/introduction) object's create operation request payload, where the object's supported fields, expressed as YAML, are the resource's supported properties. Links to documentation for each resource's corresponding payloads are listed in the table.
+
+.. tip:: The `databricks bundle validate` command returns warnings if unknown resource properties are found in bundle configuration files.
+
+
+.. list-table::
+    :header-rows: 1
+
+    * - Resource
+      - Create support
+      - Corresponding REST API object
+
+    * - [cluster](#cluster)
+      - ✓
+      - [Cluster object](/api/workspace/clusters/create)
+
+    * - [dashboard](#dashboard)
+      -
+      - [Dashboard object](/api/workspace/lakeview/create)
+
+    * - [experiment](#experiment)
+      - ✓
+      - [Experiment object](/api/workspace/experiments/createexperiment)
+
+    * - [job](#job)
+      - ✓
+      - [Job object](/api/workspace/jobs/create)
+
+    * - [model (legacy)](#model-legacy)
+      - ✓
+      - [Model (legacy) object](/api/workspace/modelregistry/createmodel)
+
+    * - [model_serving_endpoint](#model-serving-endpoint)
+      - ✓
+      - [Model serving endpoint object](/api/workspace/servingendpoints/create)
+
+    * - [pipeline](#pipeline)
+      - ✓
+      - [Pipeline object](/api/workspace/pipelines/create)
+
+    * - [quality_monitor](#quality-monitor)
+      - ✓
+      - [Quality monitor object](/api/workspace/qualitymonitors/create)
+
+    * - [registered_model](#registered-model) (<UC>)
+      - ✓
+      - [Registered model object](/api/workspace/registeredmodels/create)
+
+    * - [schema](#schema) (<UC>)
+      - ✓
+      - [Schema object](/api/workspace/schemas/create)
+
+    * - [volume](#volume) (<UC>)
+      - ✓
+      - [Volume object](/api/workspace/volumes/create)
diff --git a/bundle/internal/annotation/descriptor.go b/bundle/internal/annotation/descriptor.go
new file mode 100644
index 000000000..26c1a0b06
--- /dev/null
+++ b/bundle/internal/annotation/descriptor.go
@@ -0,0 +1,12 @@
+package annotation
+
+type Descriptor struct {
+	Description         string `json:"description,omitempty"`
+	MarkdownDescription string `json:"markdown_description,omitempty"`
+	Title               string `json:"title,omitempty"`
+	Default             any    `json:"default,omitempty"`
+	Enum                []any  `json:"enum,omitempty"`
+	MarkdownExamples    string `json:"markdown_examples,omitempty"`
+}
+
+const Placeholder = "PLACEHOLDER"
diff --git a/bundle/internal/annotation/file.go b/bundle/internal/annotation/file.go
new file mode 100644
index 000000000..0317f441a
--- /dev/null
+++ b/bundle/internal/annotation/file.go
@@ -0,0 +1,44 @@
+package annotation
+
+import (
+	"bytes"
+	"os"
+
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/databricks/cli/libs/dyn/merge"
+	"github.com/databricks/cli/libs/dyn/yamlloader"
+)
+
+// Parsed file with annotations, expected format:
+// github.com/databricks/cli/bundle/config.Bundle:
+//
+//	cluster_id:
+//	   description: "Description"
+type File map[string]map[string]Descriptor
+
+func LoadAndMerge(sources []string) (File, error) {
+	prev := dyn.NilValue
+	for _, path := range sources {
+		b, err := os.ReadFile(path)
+		if err != nil {
+			return nil, err
+		}
+		generated, err := yamlloader.LoadYAML(path, bytes.NewBuffer(b))
+		if err != nil {
+			return nil, err
+		}
+		prev, err = merge.Merge(prev, generated)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var data File
+
+	err := convert.ToTyped(&data, prev)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
diff --git a/bundle/internal/schema/annotations.go b/bundle/internal/schema/annotations.go
index 91aaa4555..ee3c25ca1 100644
--- a/bundle/internal/schema/annotations.go
+++ b/bundle/internal/schema/annotations.go
@@ -11,6 +11,7 @@ import (
 
 	yaml3 "gopkg.in/yaml.v3"
 
+	"github.com/databricks/cli/bundle/internal/annotation"
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/dyn/convert"
 	"github.com/databricks/cli/libs/dyn/merge"
@@ -19,60 +20,23 @@ import (
 	"github.com/databricks/cli/libs/jsonschema"
 )
 
-type annotation struct {
-	Description         string `json:"description,omitempty"`
-	MarkdownDescription string `json:"markdown_description,omitempty"`
-	Title               string `json:"title,omitempty"`
-	Default             any    `json:"default,omitempty"`
-	Enum                []any  `json:"enum,omitempty"`
-}
-
 type annotationHandler struct {
 	// Annotations read from all annotation files including all overrides
-	parsedAnnotations annotationFile
+	parsedAnnotations annotation.File
 	// Missing annotations for fields that are found in config that need to be added to the annotation file
-	missingAnnotations annotationFile
+	missingAnnotations annotation.File
 }
 
-/**
- * Parsed file with annotations, expected format:
- * github.com/databricks/cli/bundle/config.Bundle:
- *  	cluster_id:
- *      description: "Description"
- */
-type annotationFile map[string]map[string]annotation
-
-const Placeholder = "PLACEHOLDER"
-
 // Adds annotations to the JSON schema reading from the annotation files.
 // More details https://json-schema.org/understanding-json-schema/reference/annotations
 func newAnnotationHandler(sources []string) (*annotationHandler, error) {
-	prev := dyn.NilValue
-	for _, path := range sources {
-		b, err := os.ReadFile(path)
-		if err != nil {
-			return nil, err
-		}
-		generated, err := yamlloader.LoadYAML(path, bytes.NewBuffer(b))
-		if err != nil {
-			return nil, err
-		}
-		prev, err = merge.Merge(prev, generated)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	var data annotationFile
-
-	err := convert.ToTyped(&data, prev)
+	data, err := annotation.LoadAndMerge(sources)
 	if err != nil {
 		return nil, err
 	}
-
 	d := &annotationHandler{}
 	d.parsedAnnotations = data
-	d.missingAnnotations = annotationFile{}
+	d.missingAnnotations = annotation.File{}
 	return d, nil
 }
 
@@ -85,7 +49,7 @@ func (d *annotationHandler) addAnnotations(typ reflect.Type, s jsonschema.Schema
 
 	annotations := d.parsedAnnotations[refPath]
 	if annotations == nil {
-		annotations = map[string]annotation{}
+		annotations = map[string]annotation.Descriptor{}
 	}
 
 	rootTypeAnnotation, ok := annotations[RootTypeKey]
@@ -96,11 +60,11 @@ func (d *annotationHandler) addAnnotations(typ reflect.Type, s jsonschema.Schema
 	for k, v := range s.Properties {
 		item := annotations[k]
 		if item.Description == "" {
-			item.Description = Placeholder
+			item.Description = annotation.Placeholder
 
 			emptyAnnotations := d.missingAnnotations[refPath]
 			if emptyAnnotations == nil {
-				emptyAnnotations = map[string]annotation{}
+				emptyAnnotations = map[string]annotation.Descriptor{}
 				d.missingAnnotations[refPath] = emptyAnnotations
 			}
 			emptyAnnotations[k] = item
@@ -124,7 +88,7 @@ func (d *annotationHandler) syncWithMissingAnnotations(outputPath string) error
 	for k := range d.missingAnnotations {
 		if !isCliPath(k) {
 			delete(d.missingAnnotations, k)
-			fmt.Printf("Missing annotations for `%s` that are not in CLI package, try to fetch latest OpenAPI spec and regenerate annotations", k)
+			fmt.Printf("Missing annotations for `%s` that are not in CLI package, try to fetch latest OpenAPI spec and regenerate annotations\n", k)
 		}
 	}
 
@@ -138,7 +102,7 @@ func (d *annotationHandler) syncWithMissingAnnotations(outputPath string) error
 		return err
 	}
 
-	var outputTyped annotationFile
+	var outputTyped annotation.File
 	err = convert.ToTyped(&outputTyped, output)
 	if err != nil {
 		return err
@@ -155,8 +119,8 @@ func getPath(typ reflect.Type) string {
 	return typ.PkgPath() + "." + typ.Name()
 }
 
-func assignAnnotation(s *jsonschema.Schema, a annotation) {
-	if a.Description != Placeholder {
+func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) {
+	if a.Description != annotation.Placeholder {
 		s.Description = a.Description
 	}
 
@@ -168,7 +132,7 @@ func assignAnnotation(s *jsonschema.Schema, a annotation) {
 	s.Enum = a.Enum
 }
 
-func saveYamlWithStyle(outputPath string, annotations annotationFile) error {
+func saveYamlWithStyle(outputPath string, annotations annotation.File) error {
 	annotationOrder := yamlsaver.NewOrder([]string{"description", "markdown_description", "title", "default", "enum"})
 	style := map[string]yaml3.Style{}
 
@@ -220,15 +184,17 @@ func convertLinksToAbsoluteUrl(s string) string {
 	referencePage := "/dev-tools/bundles/reference.html"
 
 	// Regular expression to match Markdown-style links like [_](link)
-	re := regexp.MustCompile(`\[_\]\(([^)]+)\)`)
+	re := regexp.MustCompile(`\[(.*?)\]\((.*?)\)`)
 	result := re.ReplaceAllStringFunc(s, func(match string) string {
 		matches := re.FindStringSubmatch(match)
 		if len(matches) < 2 {
 			return match
 		}
-		link := matches[1]
-		var text, absoluteURL string
 
+		originalText := matches[1]
+		link := matches[2]
+
+		var text, absoluteURL string
 		if strings.HasPrefix(link, "#") {
 			text = strings.TrimPrefix(link, "#")
 			absoluteURL = fmt.Sprintf("%s%s%s", base, referencePage, link)
@@ -246,6 +212,10 @@ func convertLinksToAbsoluteUrl(s string) string {
 			return match
 		}
 
+		if originalText != "_" {
+			text = originalText
+		}
+
 		return fmt.Sprintf("[%s](%s)", text, absoluteURL)
 	})
 
diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml
index e18de7896..c10f43b04 100644
--- a/bundle/internal/schema/annotations.yml
+++ b/bundle/internal/schema/annotations.yml
@@ -1,31 +1,25 @@
 github.com/databricks/cli/bundle/config.Artifact:
   "build":
     "description": |-
-      An optional set of non-default build commands that you want to run locally before deployment.
-
-      For Python wheel builds, the Databricks CLI assumes that it can find a local install of the Python wheel package to run builds, and it runs the command python setup.py bdist_wheel by default during each bundle deployment.
-
-      To specify multiple build commands, separate each command with double-ampersand (&&) characters.
+      An optional set of non-default build commands to run locally before deployment.
   "executable":
     "description": |-
-      The executable type.
+      The executable type. Valid values are `bash`, `sh`, and `cmd`.
   "files":
     "description": |-
       The source files for the artifact.
-    "markdown_description": |-
-      The source files for the artifact, defined as an [_](#artifact_file).
   "path":
     "description": |-
       The location where the built artifact will be saved.
   "type":
     "description": |-
-      The type of the artifact.
+      Required. The type of the artifact.
     "markdown_description": |-
-      The type of the artifact. Valid values are `wheel` or `jar`
+      Required. The type of the artifact. Valid values are `whl`.
 github.com/databricks/cli/bundle/config.ArtifactFile:
   "source":
     "description": |-
-      The path of the files used to build the artifact.
+      Required. The path of the files used to build the artifact.
 github.com/databricks/cli/bundle/config.Bundle:
   "cluster_id":
     "description": |-
@@ -44,12 +38,12 @@ github.com/databricks/cli/bundle/config.Bundle:
     "description": |-
       The definition of the bundle deployment
     "markdown_description": |-
-      The definition of the bundle deployment. For supported attributes, see [_](#deployment) and [_](/dev-tools/bundles/deployment-modes.md).
+      The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md).
   "git":
     "description": |-
       The Git version control details that are associated with your bundle.
     "markdown_description": |-
-      The Git version control details that are associated with your bundle. For supported attributes, see [_](#git) and [_](/dev-tools/bundles/settings.md#git).
+      The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git).
   "name":
     "description": |-
       The name of the bundle.
@@ -63,8 +57,6 @@ github.com/databricks/cli/bundle/config.Deployment:
   "lock":
     "description": |-
       The deployment lock attributes.
-    "markdown_description": |-
-      The deployment lock attributes. See [_](#lock).
 github.com/databricks/cli/bundle/config.Experimental:
   "pydabs":
     "description": |-
@@ -74,13 +66,13 @@ github.com/databricks/cli/bundle/config.Experimental:
       Configures loading of Python code defined with 'databricks-bundles' package.
   "python_wheel_wrapper":
     "description": |-
-      Whether to use a Python wheel wrapper
+      Whether to use a Python wheel wrapper.
   "scripts":
     "description": |-
-      The commands to run
+      The commands to run.
   "use_legacy_run_as":
     "description": |-
-      Whether to use the legacy run_as behavior
+      Whether to use the legacy run_as behavior.
 github.com/databricks/cli/bundle/config.Git:
   "branch":
     "description": |-
@@ -152,66 +144,80 @@ github.com/databricks/cli/bundle/config.Resources:
       PLACEHOLDER
   "clusters":
     "description": |-
-      The cluster definitions for the bundle.
+      The cluster definitions for the bundle, where each key is the name of a cluster.
     "markdown_description": |-
-      The cluster definitions for the bundle. See [_](/dev-tools/bundles/resources.md#cluster)
+      The cluster definitions for the bundle, where each key is the name of a cluster. See [_](/dev-tools/bundles/resources.md#clusters)
   "dashboards":
     "description": |-
-      The dashboard definitions for the bundle.
+      The dashboard definitions for the bundle, where each key is the name of the dashboard.
     "markdown_description": |-
-      The dashboard definitions for the bundle. See [_](/dev-tools/bundles/resources.md#dashboard)
+      The dashboard definitions for the bundle, where each key is the name of the dashboard. See [_](/dev-tools/bundles/resources.md#dashboards)
   "experiments":
     "description": |-
-      The experiment definitions for the bundle.
+      The experiment definitions for the bundle, where each key is the name of the experiment.
     "markdown_description": |-
-      The experiment definitions for the bundle. See [_](/dev-tools/bundles/resources.md#experiment)
+      The experiment definitions for the bundle, where each key is the name of the experiment. See [_](/dev-tools/bundles/resources.md#experiments)
   "jobs":
     "description": |-
-      The job definitions for the bundle.
+      The job definitions for the bundle, where each key is the name of the job.
     "markdown_description": |-
-      The job definitions for the bundle. See [_](/dev-tools/bundles/resources.md#job)
+      The job definitions for the bundle, where each key is the name of the job. See [_](/dev-tools/bundles/resources.md#jobs)
   "model_serving_endpoints":
     "description": |-
-      The model serving endpoint definitions for the bundle.
+      The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint.
     "markdown_description": |-
-      The model serving endpoint definitions for the bundle. See [_](/dev-tools/bundles/resources.md#model_serving_endpoint)
+      The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint. See [_](/dev-tools/bundles/resources.md#model_serving_endpoints)
   "models":
     "description": |-
-      The model definitions for the bundle.
+      The model definitions for the bundle, where each key is the name of the model.
     "markdown_description": |-
-      The model definitions for the bundle. See [_](/dev-tools/bundles/resources.md#model)
+      The model definitions for the bundle, where each key is the name of the model. See [_](/dev-tools/bundles/resources.md#models)
   "pipelines":
     "description": |-
-      The pipeline definitions for the bundle.
+      The pipeline definitions for the bundle, where each key is the name of the pipeline.
     "markdown_description": |-
-      The pipeline definitions for the bundle. See [_](/dev-tools/bundles/resources.md#pipeline)
+      The pipeline definitions for the bundle, where each key is the name of the pipeline. See [_](/dev-tools/bundles/resources.md#pipelines)
   "quality_monitors":
     "description": |-
-      The quality monitor definitions for the bundle.
+      The quality monitor definitions for the bundle, where each key is the name of the quality monitor.
     "markdown_description": |-
-      The quality monitor definitions for the bundle. See [_](/dev-tools/bundles/resources.md#quality_monitor)
+      The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [_](/dev-tools/bundles/resources.md#quality_monitors)
   "registered_models":
     "description": |-
-      The registered model definitions for the bundle.
+      The registered model definitions for the bundle, where each key is the name of the <UC> registered model.
     "markdown_description": |-
-      The registered model definitions for the bundle. See [_](/dev-tools/bundles/resources.md#registered_model)
+      The registered model definitions for the bundle, where each key is the name of the <UC> registered model. See [_](/dev-tools/bundles/resources.md#registered_models)
   "schemas":
     "description": |-
-      The schema definitions for the bundle.
+      The schema definitions for the bundle, where each key is the name of the schema.
     "markdown_description": |-
-      The schema definitions for the bundle. See [_](/dev-tools/bundles/resources.md#schema)
+      The schema definitions for the bundle, where each key is the name of the schema. See [_](/dev-tools/bundles/resources.md#schemas)
   "volumes":
     "description": |-
-      PLACEHOLDER
+      The volume definitions for the bundle, where each key is the name of the volume.
+    "markdown_description": |-
+      The volume definitions for the bundle, where each key is the name of the volume. See [_](/dev-tools/bundles/resources.md#volumes)
 github.com/databricks/cli/bundle/config.Root:
   "artifacts":
     "description": |-
       Defines the attributes to build an artifact
+    "markdown_description": |-
+      Defines the attributes to build artifacts, where each key is the name of the artifact, and the value is a Map that defines the artifact build settings. For information about the `artifacts` mapping, see [_](/dev-tools/bundles/settings.md#artifacts).
+
+      Artifact settings defined in the top level of the bundle configuration can be overridden in the `targets` mapping. See [_](/dev-tools/bundles/artifact-overrides.md).
+    "markdown_examples": |-
+      ```yaml
+      artifacts:
+        default:
+          type: whl
+          build: poetry build
+          path: .
+      ```
   "bundle":
     "description": |-
-      The attributes of the bundle.
+      The bundle attributes when deploying to this target.
     "markdown_description": |-
-      The attributes of the bundle. See [_](/dev-tools/bundles/settings.md#bundle)
+      The bundle attributes when deploying to this target,
   "experimental":
     "description": |-
       Defines attributes for experimental features.
@@ -222,9 +228,21 @@ github.com/databricks/cli/bundle/config.Root:
       Specifies a list of path globs that contain configuration files to include within the bundle. See [_](/dev-tools/bundles/settings.md#include)
   "permissions":
     "description": |-
-      Defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle
+      Defines a permission for a specific entity.
     "markdown_description": |-
-      Defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle. See [_](/dev-tools/bundles/settings.md#permissions) and [_](/dev-tools/bundles/permissions.md).
+      A Sequence that defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle, where each item in the sequence is a permission for a specific entity.
+
+      See [_](/dev-tools/bundles/settings.md#permissions) and [_](/dev-tools/bundles/permissions.md).
+    "markdown_examples": |-
+      ```yaml
+      permissions:
+        - level: CAN_VIEW
+          group_name: test-group
+        - level: CAN_MANAGE
+          user_name: someone@example.com
+        - level: CAN_RUN
+          service_principal_name: 123456-abcdef
+      ```
   "presets":
     "description": |-
       Defines bundle deployment presets.
@@ -232,26 +250,39 @@ github.com/databricks/cli/bundle/config.Root:
       Defines bundle deployment presets. See [_](/dev-tools/bundles/deployment-modes.md#presets).
   "resources":
     "description": |-
-      Specifies information about the Databricks resources used by the bundle
+      A Map that defines the resources for the bundle, where each key is the name of the resource, and the value is a Map that defines the resource.
     "markdown_description": |-
-      Specifies information about the Databricks resources used by the bundle. See [_](/dev-tools/bundles/resources.md).
+      A Map that defines the resources for the bundle, where each key is the name of the resource, and the value is a Map that defines the resource. For more information about <DABS> supported resources, and resource definition reference, see [_](/dev-tools/bundles/resources.md).
+
+      ```yaml
+      resources:
+        <resource-type>:
+          <resource-name>:
+            <resource-field-name>: <resource-field-value>
+      ```
   "run_as":
     "description": |-
-      The identity to use to run the bundle.
+      The identity to use when running <DABS> workflows.
+    "markdown_description": |-
+      The identity to use when running <DABS> workflows. See [_](/dev-tools/bundles/run-as.md).
   "sync":
     "description": |-
       The files and file paths to include or exclude in the bundle.
     "markdown_description": |-
-      The files and file paths to include or exclude in the bundle. See [_](/dev-tools/bundles/)
+      The files and file paths to include or exclude in the bundle. See [_](/dev-tools/bundles/settings.md#sync).
   "targets":
     "description": |-
       Defines deployment targets for the bundle.
+    "markdown_description": |-
+      Defines deployment targets for the bundle. See [_](/dev-tools/bundles/settings.md#targets)
   "variables":
     "description": |-
       A Map that defines the custom variables for the bundle, where each key is the name of the variable, and the value is a Map that defines the variable.
   "workspace":
     "description": |-
       Defines the Databricks workspace for the bundle.
+    "markdown_description": |-
+      Defines the Databricks workspace for the bundle. See [_](/dev-tools/bundles/settings.md#workspace).
 github.com/databricks/cli/bundle/config.Sync:
   "exclude":
     "description": |-
@@ -266,11 +297,9 @@ github.com/databricks/cli/bundle/config.Target:
   "artifacts":
     "description": |-
       The artifacts to include in the target deployment.
-    "markdown_description": |-
-      The artifacts to include in the target deployment. See [_](#artifact)
   "bundle":
     "description": |-
-      The name of the bundle when deploying to this target.
+      The bundle attributes when deploying to this target.
   "cluster_id":
     "description": |-
       The ID of the cluster to use for this target.
@@ -283,8 +312,6 @@ github.com/databricks/cli/bundle/config.Target:
   "git":
     "description": |-
       The Git version control settings for the target.
-    "markdown_description": |-
-      The Git version control settings for the target. See [_](#git).
   "mode":
     "description": |-
       The deployment mode for the target.
@@ -293,38 +320,26 @@ github.com/databricks/cli/bundle/config.Target:
   "permissions":
     "description": |-
       The permissions for deploying and running the bundle in the target.
-    "markdown_description": |-
-      The permissions for deploying and running the bundle in the target. See [_](#permission).
   "presets":
     "description": |-
       The deployment presets for the target.
-    "markdown_description": |-
-      The deployment presets for the target. See [_](#preset).
   "resources":
     "description": |-
       The resource definitions for the target.
-    "markdown_description": |-
-      The resource definitions for the target. See [_](#resources).
   "run_as":
     "description": |-
       The identity to use to run the bundle.
     "markdown_description": |-
-      The identity to use to run the bundle. See [_](#job_run_as) and [_](/dev-tools/bundles/run_as.md).
+      The identity to use to run the bundle, see [_](/dev-tools/bundles/run-as.md).
   "sync":
     "description": |-
       The local paths to sync to the target workspace when a bundle is run or deployed.
-    "markdown_description": |-
-      The local paths to sync to the target workspace when a bundle is run or deployed. See [_](#sync).
   "variables":
     "description": |-
       The custom variable definitions for the target.
-    "markdown_description": |-
-      The custom variable definitions for the target. See [_](/dev-tools/bundles/settings.md#variables) and [_](/dev-tools/bundles/variables.md).
   "workspace":
     "description": |-
       The Databricks workspace for the target.
-    "markdown_description": |-
-      The Databricks workspace for the target. [_](#workspace)
 github.com/databricks/cli/bundle/config.Workspace:
   "artifact_path":
     "description": |-
@@ -374,64 +389,6 @@ github.com/databricks/cli/bundle/config.Workspace:
   "state_path":
     "description": |-
       The workspace state path
-github.com/databricks/cli/bundle/config/resources.App:
-  "active_deployment":
-    "description": |-
-      PLACEHOLDER
-  "app_status":
-    "description": |-
-      PLACEHOLDER
-  "compute_status":
-    "description": |-
-      PLACEHOLDER
-  "config":
-    "description": |-
-      PLACEHOLDER
-  "create_time":
-    "description": |-
-      PLACEHOLDER
-  "creator":
-    "description": |-
-      PLACEHOLDER
-  "default_source_code_path":
-    "description": |-
-      PLACEHOLDER
-  "description":
-    "description": |-
-      PLACEHOLDER
-  "name":
-    "description": |-
-      PLACEHOLDER
-  "pending_deployment":
-    "description": |-
-      PLACEHOLDER
-  "permissions":
-    "description": |-
-      PLACEHOLDER
-  "resources":
-    "description": |-
-      PLACEHOLDER
-  "service_principal_client_id":
-    "description": |-
-      PLACEHOLDER
-  "service_principal_id":
-    "description": |-
-      PLACEHOLDER
-  "service_principal_name":
-    "description": |-
-      PLACEHOLDER
-  "source_code_path":
-    "description": |-
-      PLACEHOLDER
-  "update_time":
-    "description": |-
-      PLACEHOLDER
-  "updater":
-    "description": |-
-      PLACEHOLDER
-  "url":
-    "description": |-
-      PLACEHOLDER
 github.com/databricks/cli/bundle/config/resources.Grant:
   "principal":
     "description": |-
@@ -440,6 +397,11 @@ github.com/databricks/cli/bundle/config/resources.Grant:
     "description": |-
       The privileges to grant to the specified entity
 github.com/databricks/cli/bundle/config/resources.Permission:
+  "-":
+    "description": |-
+      Defines a permission for a specific entity.
+    "markdown_description": |-
+      Defines a permission for a specific entity. See [_](/dev-tools/bundles/settings.md#permissions) and [_](/dev-tools/bundles/permissions.md).
   "group_name":
     "description": |-
       The name of the group that has the permission set in level.
@@ -506,6 +468,11 @@ github.com/databricks/cli/bundle/config/variable.TargetVariable:
     "description": |-
       The type of the variable.
 github.com/databricks/cli/bundle/config/variable.Variable:
+  "_":
+    "description": |-
+      Defines a custom variable for the bundle.
+    "markdown_description": |-
+      Defines a custom variable for the bundle. See [_](/dev-tools/bundles/settings.md#variables).
   "default":
     "description": |-
       PLACEHOLDER
@@ -516,107 +483,14 @@ github.com/databricks/cli/bundle/config/variable.Variable:
     "description": |-
       The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID.
     "markdown_description": |-
-      The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID."
+      The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID.
   "type":
     "description": |-
       The type of the variable.
-github.com/databricks/databricks-sdk-go/service/apps.AppDeployment:
-  "create_time":
+github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs:
+  "service_principal_name":
     "description": |-
-      PLACEHOLDER
-  "creator":
+      The application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.
+  "user_name":
     "description": |-
-      PLACEHOLDER
-  "deployment_artifacts":
-    "description": |-
-      PLACEHOLDER
-  "deployment_id":
-    "description": |-
-      PLACEHOLDER
-  "mode":
-    "description": |-
-      PLACEHOLDER
-  "source_code_path":
-    "description": |-
-      PLACEHOLDER
-  "status":
-    "description": |-
-      PLACEHOLDER
-  "update_time":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts:
-  "source_code_path":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus:
-  "message":
-    "description": |-
-      PLACEHOLDER
-  "state":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppResource:
-  "description":
-    "description": |-
-      PLACEHOLDER
-  "job":
-    "description": |-
-      PLACEHOLDER
-  "name":
-    "description": |-
-      PLACEHOLDER
-  "secret":
-    "description": |-
-      PLACEHOLDER
-  "serving_endpoint":
-    "description": |-
-      PLACEHOLDER
-  "sql_warehouse":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob:
-  "id":
-    "description": |-
-      PLACEHOLDER
-  "permission":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret:
-  "key":
-    "description": |-
-      PLACEHOLDER
-  "permission":
-    "description": |-
-      PLACEHOLDER
-  "scope":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint:
-  "name":
-    "description": |-
-      PLACEHOLDER
-  "permission":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse:
-  "id":
-    "description": |-
-      PLACEHOLDER
-  "permission":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus:
-  "message":
-    "description": |-
-      PLACEHOLDER
-  "state":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus:
-  "message":
-    "description": |-
-      PLACEHOLDER
-  "state":
-    "description": |-
-      PLACEHOLDER
+      The email of an active workspace user. Non-admin users can only set this field to their own email.
diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml
index 323432fa3..912a4fda0 100644
--- a/bundle/internal/schema/annotations_openapi_overrides.yml
+++ b/bundle/internal/schema/annotations_openapi_overrides.yml
@@ -1,4 +1,7 @@
 github.com/databricks/cli/bundle/config/resources.App:
+  "active_deployment":
+    "description": |-
+      PLACEHOLDER
   "app_status":
     "description": |-
       PLACEHOLDER
@@ -8,9 +11,30 @@ github.com/databricks/cli/bundle/config/resources.App:
   "config":
     "description": |-
       PLACEHOLDER
+  "create_time":
+    "description": |-
+      PLACEHOLDER
+  "creator":
+    "description": |-
+      PLACEHOLDER
+  "default_source_code_path":
+    "description": |-
+      PLACEHOLDER
+  "description":
+    "description": |-
+      PLACEHOLDER
+  "name":
+    "description": |-
+      PLACEHOLDER
+  "pending_deployment":
+    "description": |-
+      PLACEHOLDER
   "permissions":
     "description": |-
       PLACEHOLDER
+  "resources":
+    "description": |-
+      PLACEHOLDER
   "service_principal_client_id":
     "description": |-
       PLACEHOLDER
@@ -23,7 +47,46 @@ github.com/databricks/cli/bundle/config/resources.App:
   "source_code_path":
     "description": |-
       PLACEHOLDER
+  "update_time":
+    "description": |-
+      PLACEHOLDER
+  "updater":
+    "description": |-
+      PLACEHOLDER
+  "url":
+    "description": |-
+      PLACEHOLDER
 github.com/databricks/cli/bundle/config/resources.Cluster:
+  "_":
+    "markdown_description": |-
+      The cluster resource defines an [all-purpose cluster](/api/workspace/clusters/create).
+
+    "markdown_examples": |-
+      The following example creates a cluster named `my_cluster` and sets that as the cluster to use to run the notebook in `my_job`:
+
+      ```yaml
+      bundle:
+        name: clusters
+
+      resources:
+        clusters:
+          my_cluster:
+            num_workers: 2
+            node_type_id: "i3.xlarge"
+            autoscale:
+              min_workers: 2
+              max_workers: 7
+            spark_version: "13.3.x-scala2.12"
+            spark_conf:
+              "spark.executor.memory": "2g"
+
+        jobs:
+          my_job:
+            tasks:
+              - task_key: test_task
+                notebook_task:
+                  notebook_path: "./src/my_notebook.py"
+      ```
   "data_security_mode":
     "description": |-
       PLACEHOLDER
@@ -43,6 +106,24 @@ github.com/databricks/cli/bundle/config/resources.Cluster:
     "description": |-
       PLACEHOLDER
 github.com/databricks/cli/bundle/config/resources.Dashboard:
+  "_":
+    "markdown_description": |-
+      The dashboard resource allows you to manage [AI/BI dashboards](/api/workspace/lakeview/create) in a bundle. For information about AI/BI dashboards, see [_](/dashboards/index.md).
+    "markdown_examples": |-
+      The following example includes and deploys the sample __NYC Taxi Trip Analysis__ dashboard to the Databricks workspace.
+
+      ``` yaml
+      resources:
+        dashboards:
+          nyc_taxi_trip_analysis:
+            display_name: "NYC Taxi Trip Analysis"
+            file_path: ../src/nyc_taxi_trip_analysis.lvdash.json
+            warehouse_id: ${var.warehouse_id}
+      ```
+      If you use the UI to modify the dashboard, modifications made through the UI are not applied to the dashboard JSON file in the local bundle unless you explicitly update it using `bundle generate`. You can use the `--watch` option to continuously poll and retrieve changes to the dashboard. See [_](/dev-tools/cli/bundle-commands.md#generate).
+
+      In addition, if you attempt to deploy a bundle that contains a dashboard JSON file that is different than the one in the remote workspace, an error will occur. To force the deploy and overwrite the dashboard in the remote workspace with the local one, use the `--force` option. See [_](/dev-tools/cli/bundle-commands.md#deploy).
+
   "embed_credentials":
     "description": |-
       PLACEHOLDER
@@ -53,6 +134,24 @@ github.com/databricks/cli/bundle/config/resources.Dashboard:
     "description": |-
       PLACEHOLDER
 github.com/databricks/cli/bundle/config/resources.Job:
+  "_":
+    "markdown_description": |-
+      The job resource allows you to define [jobs and their corresponding tasks](/api/workspace/jobs/create) in your bundle. For information about jobs, see [_](/jobs/index.md). For a tutorial that uses a <DABS> template to create a job, see [_](/dev-tools/bundles/jobs-tutorial.md).
+    "markdown_examples": |-
+      The following example defines a job with the resource key `hello-job` with one notebook task:
+
+      ```yaml
+      resources:
+        jobs:
+          hello-job:
+            name: hello-job
+            tasks:
+              - task_key: hello-task
+                notebook_task:
+                  notebook_path: ./hello.py
+      ```
+
+      For information about defining job tasks and overriding job settings, see [_](/dev-tools/bundles/job-task-types.md), [_](/dev-tools/bundles/job-task-override.md), and [_](/dev-tools/bundles/cluster-override.md).
   "health":
     "description": |-
       PLACEHOLDER
@@ -63,30 +162,186 @@ github.com/databricks/cli/bundle/config/resources.Job:
     "description": |-
       PLACEHOLDER
 github.com/databricks/cli/bundle/config/resources.MlflowExperiment:
+  "_":
+    "markdown_description": |-
+      The experiment resource allows you to define [MLflow experiments](/api/workspace/experiments/createexperiment) in a bundle. For information about MLflow experiments, see [_](/mlflow/experiments.md).
+    "markdown_examples": |-
+      The following example defines an experiment that all users can view:
+
+      ```yaml
+      resources:
+        experiments:
+          experiment:
+            name: my_ml_experiment
+            permissions:
+              - level: CAN_READ
+                group_name: users
+            description: MLflow experiment used to track runs
+      ```
   "permissions":
     "description": |-
       PLACEHOLDER
 github.com/databricks/cli/bundle/config/resources.MlflowModel:
+  "_":
+    "markdown_description": |-
+      The model resource allows you to define [legacy models](/api/workspace/modelregistry/createmodel) in bundles. Databricks recommends you use <UC> [registered models](#registered-model) instead.
   "permissions":
     "description": |-
       PLACEHOLDER
 github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint:
+  "_":
+    "markdown_description": |-
+      The model_serving_endpoint resource allows you to define [model serving endpoints](/api/workspace/servingendpoints/create). See [_](/machine-learning/model-serving/manage-serving-endpoints.md).
+    "markdown_examples": |-
+      The following example defines a <UC> model serving endpoint:
+
+      ```yaml
+      resources:
+        model_serving_endpoints:
+          uc_model_serving_endpoint:
+            name: "uc-model-endpoint"
+            config:
+              served_entities:
+              - entity_name: "myCatalog.mySchema.my-ads-model"
+                entity_version: "10"
+                workload_size: "Small"
+                scale_to_zero_enabled: "true"
+              traffic_config:
+                routes:
+                - served_model_name: "my-ads-model-10"
+                  traffic_percentage: "100"
+            tags:
+            - key: "team"
+              value: "data science"
+      ```
   "permissions":
     "description": |-
       PLACEHOLDER
 github.com/databricks/cli/bundle/config/resources.Pipeline:
+  "_":
+    "markdown_description": |-
+      The pipeline resource allows you to create <DLT> [pipelines](/api/workspace/pipelines/create). For information about pipelines, see [_](/delta-live-tables/index.md). For a tutorial that uses the <DABS> template to create a pipeline, see [_](/dev-tools/bundles/pipelines-tutorial.md).
+    "markdown_examples": |-
+      The following example defines a pipeline with the resource key `hello-pipeline`:
+
+      ```yaml
+      resources:
+        pipelines:
+          hello-pipeline:
+            name: hello-pipeline
+            clusters:
+              - label: default
+                num_workers: 1
+            development: true
+            continuous: false
+            channel: CURRENT
+            edition: CORE
+            photon: false
+            libraries:
+              - notebook:
+                  path: ./pipeline.py
+      ```
   "permissions":
     "description": |-
       PLACEHOLDER
 github.com/databricks/cli/bundle/config/resources.QualityMonitor:
+  "_":
+    "markdown_description": |-
+      The quality_monitor resource allows you to define a <UC> [table monitor](/api/workspace/qualitymonitors/create). For information about monitors, see [_](/machine-learning/model-serving/monitor-diagnose-endpoints.md).
+    "markdown_examples": |-
+      The following example defines a quality monitor:
+
+      ```yaml
+      resources:
+        quality_monitors:
+          my_quality_monitor:
+            table_name: dev.mlops_schema.predictions
+            output_schema_name: ${bundle.target}.mlops_schema
+            assets_dir: /Users/${workspace.current_user.userName}/databricks_lakehouse_monitoring
+            inference_log:
+              granularities: [1 day]
+              model_id_col: model_id
+              prediction_col: prediction
+              label_col: price
+              problem_type: PROBLEM_TYPE_REGRESSION
+              timestamp_col: timestamp
+            schedule:
+              quartz_cron_expression: 0 0 8 * * ? # Run Every day at 8am
+              timezone_id: UTC
+      ```
   "table_name":
     "description": |-
       PLACEHOLDER
 github.com/databricks/cli/bundle/config/resources.RegisteredModel:
+  "_":
+    "markdown_description": |-
+      The registered model resource allows you to define models in <UC>. For information about <UC> [registered models](/api/workspace/registeredmodels/create), see [_](/machine-learning/manage-model-lifecycle/index.md).
+    "markdown_examples": |-
+      The following example defines a registered model in <UC>:
+
+      ```yaml
+      resources:
+        registered_models:
+            model:
+              name: my_model
+              catalog_name: ${bundle.target}
+              schema_name: mlops_schema
+              comment: Registered model in Unity Catalog for ${bundle.target} deployment target
+              grants:
+                - privileges:
+                    - EXECUTE
+                  principal: account users
+      ```
   "grants":
     "description": |-
       PLACEHOLDER
 github.com/databricks/cli/bundle/config/resources.Schema:
+  "_":
+    "markdown_description": |-
+      The schema resource type allows you to define <UC> [schemas](/api/workspace/schemas/create) for tables and other assets in your workflows and pipelines created as part of a bundle. A schema, different from other resource types, has the following limitations:
+
+      - The owner of a schema resource is always the deployment user, and cannot be changed. If `run_as` is specified in the bundle, it will be ignored by operations on the schema.
+      - Only fields supported by the corresponding [Schemas object create API](/api/workspace/schemas/create) are available for the schema resource. For example, `enable_predictive_optimization` is not supported as it is only available on the [update API](/api/workspace/schemas/update).
+    "markdown_examples": |-
+      The following example defines a pipeline with the resource key `my_pipeline` that creates a <UC> schema with the key `my_schema` as the target:
+
+      ```yaml
+      resources:
+        pipelines:
+          my_pipeline:
+            name: test-pipeline-{{.unique_id}}
+            libraries:
+              - notebook:
+                  path: ./nb.sql
+            development: true
+            catalog: main
+            target: ${resources.schemas.my_schema.id}
+
+        schemas:
+          my_schema:
+            name: test-schema-{{.unique_id}}
+            catalog_name: main
+            comment: This schema was created by DABs.
+      ```
+
+      A top-level grants mapping is not supported by <DABS>, so if you want to set grants for a schema, define the grants for the schema within the `schemas` mapping. For more information about grants, see [_](/data-governance/unity-catalog/manage-privileges/index.md#grant).
+
+      The following example defines a <UC> schema with grants:
+
+      ```yaml
+      resources:
+        schemas:
+          my_schema:
+            name: test-schema
+            grants:
+              - principal: users
+                privileges:
+                  - CAN_MANAGE
+              - principal: my_team
+                privileges:
+                  - CAN_READ
+            catalog_name: main
+        ```
   "grants":
     "description": |-
       PLACEHOLDER
@@ -94,6 +349,27 @@ github.com/databricks/cli/bundle/config/resources.Schema:
     "description": |-
       PLACEHOLDER
 github.com/databricks/cli/bundle/config/resources.Volume:
+  "_":
+    "markdown_description": |-
+      The volume resource type allows you to define and create <UC> [volumes](/api/workspace/volumes/create) as part of a bundle. When deploying a bundle with a volume defined, note that:
+
+      - A volume cannot be referenced in the `artifact_path` for the bundle until it exists in the workspace. Hence, if you want to use <DABS> to create the volume, you must first define the volume in the bundle, deploy it to create the volume, then reference it in the `artifact_path` in subsequent deployments.
+
+      - Volumes in the bundle are not prepended with the `dev_${workspace.current_user.short_name}` prefix when the deployment target has `mode: development` configured. However, you can manually configure this prefix. See [_](/dev-tools/bundles/deployment-modes.md#custom-presets).
+
+    "markdown_examples": |-
+      The following example creates a <UC> volume with the key `my_volume`:
+
+      ```yaml
+      resources:
+        volumes:
+          my_volume:
+            catalog_name: main
+            name: my_volume
+            schema_name: my_schema
+      ```
+
+      For an example bundle that runs a job that writes to a file in <UC> volume, see the [bundle-examples GitHub repository](https://github.com/databricks/bundle-examples/tree/main/knowledge_base/write_from_job_to_volume).
   "grants":
     "description": |-
       PLACEHOLDER
@@ -197,6 +473,85 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger:
   "manual":
     "description": |-
       PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppDeployment:
+  "create_time":
+    "description": |-
+      PLACEHOLDER
+  "creator":
+    "description": |-
+      PLACEHOLDER
+  "deployment_artifacts":
+    "description": |-
+      PLACEHOLDER
+  "deployment_id":
+    "description": |-
+      PLACEHOLDER
+  "mode":
+    "description": |-
+      PLACEHOLDER
+  "source_code_path":
+    "description": |-
+      PLACEHOLDER
+  "status":
+    "description": |-
+      PLACEHOLDER
+  "update_time":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts:
+  "source_code_path":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus:
+  "message":
+    "description": |-
+      PLACEHOLDER
+  "state":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob:
+  "id":
+    "description": |-
+      PLACEHOLDER
+  "permission":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret:
+  "key":
+    "description": |-
+      PLACEHOLDER
+  "permission":
+    "description": |-
+      PLACEHOLDER
+  "scope":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint:
+  "name":
+    "description": |-
+      PLACEHOLDER
+  "permission":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse:
+  "id":
+    "description": |-
+      PLACEHOLDER
+  "permission":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus:
+  "message":
+    "description": |-
+      PLACEHOLDER
+  "state":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus:
+  "message":
+    "description": |-
+      PLACEHOLDER
+  "state":
 github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput:
   "entity_version":
     "description": |-
diff --git a/bundle/internal/schema/annotations_test.go b/bundle/internal/schema/annotations_test.go
index d7e2fea7c..0e1593359 100644
--- a/bundle/internal/schema/annotations_test.go
+++ b/bundle/internal/schema/annotations_test.go
@@ -33,6 +33,10 @@ func TestConvertLinksToAbsoluteUrl(t *testing.T) {
 			input:    "This is a link to [external](https://external.com)",
 			expected: "This is a link to [external](https://external.com)",
 		},
+		{
+			input:    "This is a link to [one](/relative), [two](/relative-2)",
+			expected: "This is a link to [one](https://docs.databricks.com/relative), [two](https://docs.databricks.com/relative-2)",
+		},
 	}
 
 	for _, test := range tests {
diff --git a/bundle/internal/schema/main_test.go b/bundle/internal/schema/main_test.go
index 06e89c856..051243c4d 100644
--- a/bundle/internal/schema/main_test.go
+++ b/bundle/internal/schema/main_test.go
@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/internal/annotation"
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/dyn/merge"
 	"github.com/databricks/cli/libs/dyn/yamlloader"
@@ -113,13 +114,13 @@ func TestNoDetachedAnnotations(t *testing.T) {
 	assert.Empty(t, types, "Detached annotations found, regenerate schema and check for package path changes")
 }
 
-func getAnnotations(path string) (annotationFile, error) {
+func getAnnotations(path string) (annotation.File, error) {
 	b, err := os.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}
 
-	var data annotationFile
+	var data annotation.File
 	err = yaml.Unmarshal(b, &data)
 	return data, err
 }
diff --git a/bundle/internal/schema/parser.go b/bundle/internal/schema/parser.go
index 919908429..50e69e7c8 100644
--- a/bundle/internal/schema/parser.go
+++ b/bundle/internal/schema/parser.go
@@ -8,6 +8,7 @@ import (
 	"reflect"
 	"strings"
 
+	"github.com/databricks/cli/bundle/internal/annotation"
 	"github.com/databricks/cli/libs/jsonschema"
 	"gopkg.in/yaml.v3"
 )
@@ -114,8 +115,8 @@ func mapIncorrectTypNames(ref string) string {
 
 // Use the OpenAPI spec to load descriptions for the given type.
 func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overridesPath string) error {
-	annotations := annotationFile{}
-	overrides := annotationFile{}
+	annotations := annotation.File{}
+	overrides := annotation.File{}
 
 	b, err := os.ReadFile(overridesPath)
 	if err != nil {
@@ -126,7 +127,7 @@ func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overrid
 		return err
 	}
 	if overrides == nil {
-		overrides = annotationFile{}
+		overrides = annotation.File{}
 	}
 
 	_, err = jsonschema.FromType(typ, []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{
@@ -137,16 +138,16 @@ func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overrid
 			}
 
 			basePath := getPath(typ)
-			pkg := map[string]annotation{}
+			pkg := map[string]annotation.Descriptor{}
 			annotations[basePath] = pkg
 
 			if ref.Description != "" || ref.Enum != nil {
-				pkg[RootTypeKey] = annotation{Description: ref.Description, Enum: ref.Enum}
+				pkg[RootTypeKey] = annotation.Descriptor{Description: ref.Description, Enum: ref.Enum}
 			}
 
 			for k := range s.Properties {
 				if refProp, ok := ref.Properties[k]; ok {
-					pkg[k] = annotation{Description: refProp.Description, Enum: refProp.Enum}
+					pkg[k] = annotation.Descriptor{Description: refProp.Description, Enum: refProp.Enum}
 					if refProp.Description == "" {
 						addEmptyOverride(k, basePath, overrides)
 					}
@@ -195,22 +196,22 @@ func prependCommentToFile(outputPath, comment string) error {
 	return err
 }
 
-func addEmptyOverride(key, pkg string, overridesFile annotationFile) {
+func addEmptyOverride(key, pkg string, overridesFile annotation.File) {
 	if overridesFile[pkg] == nil {
-		overridesFile[pkg] = map[string]annotation{}
+		overridesFile[pkg] = map[string]annotation.Descriptor{}
 	}
 
 	overrides := overridesFile[pkg]
 	if overrides[key].Description == "" {
-		overrides[key] = annotation{Description: Placeholder}
+		overrides[key] = annotation.Descriptor{Description: annotation.Placeholder}
 	}
 
 	a, ok := overrides[key]
 	if !ok {
-		a = annotation{}
+		a = annotation.Descriptor{}
 	}
 	if a.Description == "" {
-		a.Description = Placeholder
+		a.Description = annotation.Placeholder
 	}
 	overrides[key] = a
 }
diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json
index 17a621ba0..7c72c440e 100644
--- a/bundle/schema/jsonschema.json
+++ b/bundle/schema/jsonschema.json
@@ -258,7 +258,8 @@
                         "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.WorkloadType"
                       }
                     },
-                    "additionalProperties": false
+                    "additionalProperties": false,
+                    "markdownDescription": "The cluster resource defines an [all-purpose cluster](https://docs.databricks.com/api/workspace/clusters/create)."
                   },
                   {
                     "type": "string",
@@ -321,7 +322,8 @@
                         "$ref": "#/$defs/string"
                       }
                     },
-                    "additionalProperties": false
+                    "additionalProperties": false,
+                    "markdownDescription": "The dashboard resource allows you to manage [AI/BI dashboards](https://docs.databricks.com/api/workspace/lakeview/create) in a bundle. For information about AI/BI dashboards, see [link](https://docs.databricks.com/dashboards/index.html)."
                   },
                   {
                     "type": "string",
@@ -442,7 +444,8 @@
                         "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.WebhookNotifications"
                       }
                     },
-                    "additionalProperties": false
+                    "additionalProperties": false,
+                    "markdownDescription": "The job resource allows you to define [jobs and their corresponding tasks](https://docs.databricks.com/api/workspace/jobs/create) in your bundle. For information about jobs, see [link](https://docs.databricks.com/jobs/index.html). For a tutorial that uses a \u003cDABS\u003e template to create a job, see [link](https://docs.databricks.com/dev-tools/bundles/jobs-tutorial.html)."
                   },
                   {
                     "type": "string",
@@ -487,7 +490,8 @@
                         "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.ExperimentTag"
                       }
                     },
-                    "additionalProperties": false
+                    "additionalProperties": false,
+                    "markdownDescription": "The experiment resource allows you to define [MLflow experiments](https://docs.databricks.com/api/workspace/experiments/createexperiment) in a bundle. For information about MLflow experiments, see [link](https://docs.databricks.com/mlflow/experiments.html)."
                   },
                   {
                     "type": "string",
@@ -532,7 +536,8 @@
                         "$ref": "#/$defs/string"
                       }
                     },
-                    "additionalProperties": false
+                    "additionalProperties": false,
+                    "markdownDescription": "The model resource allows you to define [legacy models](https://docs.databricks.com/api/workspace/modelregistry/createmodel) in bundles. Databricks recommends you use \u003cUC\u003e [registered models](https://docs.databricks.com/dev-tools/bundles/reference.html#registered-model) instead."
                   },
                   {
                     "type": "string",
@@ -576,7 +581,8 @@
                     "additionalProperties": false,
                     "required": [
                       "name"
-                    ]
+                    ],
+                    "markdownDescription": "The model_serving_endpoint resource allows you to define [model serving endpoints](https://docs.databricks.com/api/workspace/servingendpoints/create). See [link](https://docs.databricks.com/machine-learning/model-serving/manage-serving-endpoints.html)."
                   },
                   {
                     "type": "string",
@@ -718,7 +724,8 @@
                         "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger"
                       }
                     },
-                    "additionalProperties": false
+                    "additionalProperties": false,
+                    "markdownDescription": "The pipeline resource allows you to create \u003cDLT\u003e [pipelines](https://docs.databricks.com/api/workspace/pipelines/create). For information about pipelines, see [link](https://docs.databricks.com/delta-live-tables/index.html). For a tutorial that uses the \u003cDABS\u003e template to create a pipeline, see [link](https://docs.databricks.com/dev-tools/bundles/pipelines-tutorial.html)."
                   },
                   {
                     "type": "string",
@@ -792,7 +799,8 @@
                       "table_name",
                       "assets_dir",
                       "output_schema_name"
-                    ]
+                    ],
+                    "markdownDescription": "The quality_monitor resource allows you to define a \u003cUC\u003e [table monitor](https://docs.databricks.com/api/workspace/qualitymonitors/create). For information about monitors, see [link](https://docs.databricks.com/machine-learning/model-serving/monitor-diagnose-endpoints.html)."
                   },
                   {
                     "type": "string",
@@ -834,7 +842,8 @@
                       "catalog_name",
                       "name",
                       "schema_name"
-                    ]
+                    ],
+                    "markdownDescription": "The registered model resource allows you to define models in \u003cUC\u003e. For information about \u003cUC\u003e [registered models](https://docs.databricks.com/api/workspace/registeredmodels/create), see [link](https://docs.databricks.com/machine-learning/manage-model-lifecycle/index.html)."
                   },
                   {
                     "type": "string",
@@ -874,7 +883,8 @@
                     "required": [
                       "catalog_name",
                       "name"
-                    ]
+                    ],
+                    "markdownDescription": "The schema resource type allows you to define \u003cUC\u003e [schemas](https://docs.databricks.com/api/workspace/schemas/create) for tables and other assets in your workflows and pipelines created as part of a bundle. A schema, different from other resource types, has the following limitations:\n\n- The owner of a schema resource is always the deployment user, and cannot be changed. If `run_as` is specified in the bundle, it will be ignored by operations on the schema.\n- Only fields supported by the corresponding [Schemas object create API](https://docs.databricks.com/api/workspace/schemas/create) are available for the schema resource. For example, `enable_predictive_optimization` is not supported as it is only available on the [update API](https://docs.databricks.com/api/workspace/schemas/update)."
                   },
                   {
                     "type": "string",
@@ -919,7 +929,8 @@
                       "catalog_name",
                       "name",
                       "schema_name"
-                    ]
+                    ],
+                    "markdownDescription": "The volume resource type allows you to define and create \u003cUC\u003e [volumes](https://docs.databricks.com/api/workspace/volumes/create) as part of a bundle. When deploying a bundle with a volume defined, note that:\n\n- A volume cannot be referenced in the `artifact_path` for the bundle until it exists in the workspace. Hence, if you want to use \u003cDABS\u003e to create the volume, you must first define the volume in the bundle, deploy it to create the volume, then reference it in the `artifact_path` in subsequent deployments.\n\n- Volumes in the bundle are not prepended with the `dev_${workspace.current_user.short_name}` prefix when the deployment target has `mode: development` configured. However, you can manually configure this prefix. See [custom-presets](https://docs.databricks.com/dev-tools/bundles/deployment-modes.html#custom-presets)."
                   },
                   {
                     "type": "string",
@@ -1005,6 +1016,7 @@
               },
               "variable.Variable": {
                 "type": "object",
+                "description": "Defines a custom variable for the bundle.",
                 "properties": {
                   "default": {
                     "$ref": "#/$defs/interface"
@@ -1016,14 +1028,15 @@
                   "lookup": {
                     "description": "The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID.",
                     "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.Lookup",
-                    "markdownDescription": "The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID.\""
+                    "markdownDescription": "The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID."
                   },
                   "type": {
                     "description": "The type of the variable.",
                     "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.VariableType"
                   }
                 },
-                "additionalProperties": false
+                "additionalProperties": false,
+                "markdownDescription": "Defines a custom variable for the bundle. See [variables](https://docs.databricks.com/dev-tools/bundles/settings.html#variables)."
               },
               "variable.VariableType": {
                 "type": "string"
@@ -1035,26 +1048,25 @@
                   "type": "object",
                   "properties": {
                     "build": {
-                      "description": "An optional set of non-default build commands that you want to run locally before deployment.\n\nFor Python wheel builds, the Databricks CLI assumes that it can find a local install of the Python wheel package to run builds, and it runs the command python setup.py bdist_wheel by default during each bundle deployment.\n\nTo specify multiple build commands, separate each command with double-ampersand (\u0026\u0026) characters.",
+                      "description": "An optional set of non-default build commands to run locally before deployment.",
                       "$ref": "#/$defs/string"
                     },
                     "executable": {
-                      "description": "The executable type.",
+                      "description": "The executable type. Valid values are `bash`, `sh`, and `cmd`.",
                       "$ref": "#/$defs/github.com/databricks/cli/libs/exec.ExecutableType"
                     },
                     "files": {
                       "description": "The source files for the artifact.",
-                      "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config.ArtifactFile",
-                      "markdownDescription": "The source files for the artifact, defined as an [artifact_file](https://docs.databricks.com/dev-tools/bundles/reference.html#artifact_file)."
+                      "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config.ArtifactFile"
                     },
                     "path": {
                       "description": "The location where the built artifact will be saved.",
                       "$ref": "#/$defs/string"
                     },
                     "type": {
-                      "description": "The type of the artifact.",
+                      "description": "Required. The type of the artifact.",
                       "$ref": "#/$defs/github.com/databricks/cli/bundle/config.ArtifactType",
-                      "markdownDescription": "The type of the artifact. Valid values are `wheel` or `jar`"
+                      "markdownDescription": "Required. The type of the artifact. Valid values are `whl`."
                     }
                   },
                   "additionalProperties": false,
@@ -1074,7 +1086,7 @@
                   "type": "object",
                   "properties": {
                     "source": {
-                      "description": "The path of the files used to build the artifact.",
+                      "description": "Required. The path of the files used to build the artifact.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -1113,12 +1125,12 @@
                     "deployment": {
                       "description": "The definition of the bundle deployment",
                       "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Deployment",
-                      "markdownDescription": "The definition of the bundle deployment. For supported attributes, see [deployment](https://docs.databricks.com/dev-tools/bundles/reference.html#deployment) and [link](https://docs.databricks.com/dev-tools/bundles/deployment-modes.html)."
+                      "markdownDescription": "The definition of the bundle deployment. For supported attributes see [link](https://docs.databricks.com/dev-tools/bundles/deployment-modes.html)."
                     },
                     "git": {
                       "description": "The Git version control details that are associated with your bundle.",
                       "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Git",
-                      "markdownDescription": "The Git version control details that are associated with your bundle. For supported attributes, see [git](https://docs.databricks.com/dev-tools/bundles/reference.html#git) and [git](https://docs.databricks.com/dev-tools/bundles/settings.html#git)."
+                      "markdownDescription": "The Git version control details that are associated with your bundle. For supported attributes see [git](https://docs.databricks.com/dev-tools/bundles/settings.html#git)."
                     },
                     "name": {
                       "description": "The name of the bundle.",
@@ -1154,8 +1166,7 @@
                     },
                     "lock": {
                       "description": "The deployment lock attributes.",
-                      "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Lock",
-                      "markdownDescription": "The deployment lock attributes. See [lock](https://docs.databricks.com/dev-tools/bundles/reference.html#lock)."
+                      "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Lock"
                     }
                   },
                   "additionalProperties": false
@@ -1180,15 +1191,15 @@
                       "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Python"
                     },
                     "python_wheel_wrapper": {
-                      "description": "Whether to use a Python wheel wrapper",
+                      "description": "Whether to use a Python wheel wrapper.",
                       "$ref": "#/$defs/bool"
                     },
                     "scripts": {
-                      "description": "The commands to run",
+                      "description": "The commands to run.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Command"
                     },
                     "use_legacy_run_as": {
-                      "description": "Whether to use the legacy run_as behavior",
+                      "description": "Whether to use the legacy run_as behavior.",
                       "$ref": "#/$defs/bool"
                     }
                   },
@@ -1352,57 +1363,59 @@
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.App"
                     },
                     "clusters": {
-                      "description": "The cluster definitions for the bundle.",
+                      "description": "The cluster definitions for the bundle, where each key is the name of a cluster.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Cluster",
-                      "markdownDescription": "The cluster definitions for the bundle. See [cluster](https://docs.databricks.com/dev-tools/bundles/resources.html#cluster)"
+                      "markdownDescription": "The cluster definitions for the bundle, where each key is the name of a cluster. See [clusters](https://docs.databricks.com/dev-tools/bundles/resources.html#clusters)"
                     },
                     "dashboards": {
-                      "description": "The dashboard definitions for the bundle.",
+                      "description": "The dashboard definitions for the bundle, where each key is the name of the dashboard.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Dashboard",
-                      "markdownDescription": "The dashboard definitions for the bundle. See [dashboard](https://docs.databricks.com/dev-tools/bundles/resources.html#dashboard)"
+                      "markdownDescription": "The dashboard definitions for the bundle, where each key is the name of the dashboard. See [dashboards](https://docs.databricks.com/dev-tools/bundles/resources.html#dashboards)"
                     },
                     "experiments": {
-                      "description": "The experiment definitions for the bundle.",
+                      "description": "The experiment definitions for the bundle, where each key is the name of the experiment.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MlflowExperiment",
-                      "markdownDescription": "The experiment definitions for the bundle. See [experiment](https://docs.databricks.com/dev-tools/bundles/resources.html#experiment)"
+                      "markdownDescription": "The experiment definitions for the bundle, where each key is the name of the experiment. See [experiments](https://docs.databricks.com/dev-tools/bundles/resources.html#experiments)"
                     },
                     "jobs": {
-                      "description": "The job definitions for the bundle.",
+                      "description": "The job definitions for the bundle, where each key is the name of the job.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job",
-                      "markdownDescription": "The job definitions for the bundle. See [job](https://docs.databricks.com/dev-tools/bundles/resources.html#job)"
+                      "markdownDescription": "The job definitions for the bundle, where each key is the name of the job. See [jobs](https://docs.databricks.com/dev-tools/bundles/resources.html#jobs)"
                     },
                     "model_serving_endpoints": {
-                      "description": "The model serving endpoint definitions for the bundle.",
+                      "description": "The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint",
-                      "markdownDescription": "The model serving endpoint definitions for the bundle. See [model_serving_endpoint](https://docs.databricks.com/dev-tools/bundles/resources.html#model_serving_endpoint)"
+                      "markdownDescription": "The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint. See [model_serving_endpoints](https://docs.databricks.com/dev-tools/bundles/resources.html#model_serving_endpoints)"
                     },
                     "models": {
-                      "description": "The model definitions for the bundle.",
+                      "description": "The model definitions for the bundle, where each key is the name of the model.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MlflowModel",
-                      "markdownDescription": "The model definitions for the bundle. See [model](https://docs.databricks.com/dev-tools/bundles/resources.html#model)"
+                      "markdownDescription": "The model definitions for the bundle, where each key is the name of the model. See [models](https://docs.databricks.com/dev-tools/bundles/resources.html#models)"
                     },
                     "pipelines": {
-                      "description": "The pipeline definitions for the bundle.",
+                      "description": "The pipeline definitions for the bundle, where each key is the name of the pipeline.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Pipeline",
-                      "markdownDescription": "The pipeline definitions for the bundle. See [pipeline](https://docs.databricks.com/dev-tools/bundles/resources.html#pipeline)"
+                      "markdownDescription": "The pipeline definitions for the bundle, where each key is the name of the pipeline. See [pipelines](https://docs.databricks.com/dev-tools/bundles/resources.html#pipelines)"
                     },
                     "quality_monitors": {
-                      "description": "The quality monitor definitions for the bundle.",
+                      "description": "The quality monitor definitions for the bundle, where each key is the name of the quality monitor.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.QualityMonitor",
-                      "markdownDescription": "The quality monitor definitions for the bundle. See [quality_monitor](https://docs.databricks.com/dev-tools/bundles/resources.html#quality_monitor)"
+                      "markdownDescription": "The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [quality_monitors](https://docs.databricks.com/dev-tools/bundles/resources.html#quality_monitors)"
                     },
                     "registered_models": {
-                      "description": "The registered model definitions for the bundle.",
+                      "description": "The registered model definitions for the bundle, where each key is the name of the \u003cUC\u003e registered model.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.RegisteredModel",
-                      "markdownDescription": "The registered model definitions for the bundle. See [registered_model](https://docs.databricks.com/dev-tools/bundles/resources.html#registered_model)"
+                      "markdownDescription": "The registered model definitions for the bundle, where each key is the name of the \u003cUC\u003e registered model. See [registered_models](https://docs.databricks.com/dev-tools/bundles/resources.html#registered_models)"
                     },
                     "schemas": {
-                      "description": "The schema definitions for the bundle.",
+                      "description": "The schema definitions for the bundle, where each key is the name of the schema.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Schema",
-                      "markdownDescription": "The schema definitions for the bundle. See [schema](https://docs.databricks.com/dev-tools/bundles/resources.html#schema)"
+                      "markdownDescription": "The schema definitions for the bundle, where each key is the name of the schema. See [schemas](https://docs.databricks.com/dev-tools/bundles/resources.html#schemas)"
                     },
                     "volumes": {
-                      "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Volume"
+                      "description": "The volume definitions for the bundle, where each key is the name of the volume.",
+                      "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Volume",
+                      "markdownDescription": "The volume definitions for the bundle, where each key is the name of the volume. See [volumes](https://docs.databricks.com/dev-tools/bundles/resources.html#volumes)"
                     }
                   },
                   "additionalProperties": false
@@ -1446,11 +1459,10 @@
                   "properties": {
                     "artifacts": {
                       "description": "The artifacts to include in the target deployment.",
-                      "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact",
-                      "markdownDescription": "The artifacts to include in the target deployment. See [artifact](https://docs.databricks.com/dev-tools/bundles/reference.html#artifact)"
+                      "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact"
                     },
                     "bundle": {
-                      "description": "The name of the bundle when deploying to this target.",
+                      "description": "The bundle attributes when deploying to this target.",
                       "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Bundle"
                     },
                     "cluster_id": {
@@ -1467,8 +1479,7 @@
                     },
                     "git": {
                       "description": "The Git version control settings for the target.",
-                      "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Git",
-                      "markdownDescription": "The Git version control settings for the target. See [git](https://docs.databricks.com/dev-tools/bundles/reference.html#git)."
+                      "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Git"
                     },
                     "mode": {
                       "description": "The deployment mode for the target.",
@@ -1477,38 +1488,32 @@
                     },
                     "permissions": {
                       "description": "The permissions for deploying and running the bundle in the target.",
-                      "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission",
-                      "markdownDescription": "The permissions for deploying and running the bundle in the target. See [permission](https://docs.databricks.com/dev-tools/bundles/reference.html#permission)."
+                      "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission"
                     },
                     "presets": {
                       "description": "The deployment presets for the target.",
-                      "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Presets",
-                      "markdownDescription": "The deployment presets for the target. See [preset](https://docs.databricks.com/dev-tools/bundles/reference.html#preset)."
+                      "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Presets"
                     },
                     "resources": {
                       "description": "The resource definitions for the target.",
-                      "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Resources",
-                      "markdownDescription": "The resource definitions for the target. See [resources](https://docs.databricks.com/dev-tools/bundles/reference.html#resources)."
+                      "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Resources"
                     },
                     "run_as": {
                       "description": "The identity to use to run the bundle.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs",
-                      "markdownDescription": "The identity to use to run the bundle. See [job_run_as](https://docs.databricks.com/dev-tools/bundles/reference.html#job_run_as) and [link](https://docs.databricks.com/dev-tools/bundles/run_as.html)."
+                      "markdownDescription": "The identity to use to run the bundle, see [link](https://docs.databricks.com/dev-tools/bundles/run-as.html)."
                     },
                     "sync": {
                       "description": "The local paths to sync to the target workspace when a bundle is run or deployed.",
-                      "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Sync",
-                      "markdownDescription": "The local paths to sync to the target workspace when a bundle is run or deployed. See [sync](https://docs.databricks.com/dev-tools/bundles/reference.html#sync)."
+                      "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Sync"
                     },
                     "variables": {
                       "description": "The custom variable definitions for the target.",
-                      "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/variable.TargetVariable",
-                      "markdownDescription": "The custom variable definitions for the target. See [variables](https://docs.databricks.com/dev-tools/bundles/settings.html#variables) and [link](https://docs.databricks.com/dev-tools/bundles/variables.html)."
+                      "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/variable.TargetVariable"
                     },
                     "workspace": {
                       "description": "The Databricks workspace for the target.",
-                      "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace",
-                      "markdownDescription": "The Databricks workspace for the target. [workspace](https://docs.databricks.com/dev-tools/bundles/reference.html#workspace)"
+                      "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace"
                     }
                   },
                   "additionalProperties": false
@@ -1719,12 +1724,14 @@
                   "type": "object",
                   "properties": {
                     "description": {
+                      "description": "Description of the App Resource.",
                       "$ref": "#/$defs/string"
                     },
                     "job": {
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob"
                     },
                     "name": {
+                      "description": "Name of the App Resource.",
                       "$ref": "#/$defs/string"
                     },
                     "secret": {
@@ -1980,6 +1987,7 @@
                       "$ref": "#/$defs/string"
                     },
                     "state": {
+                      "description": "State of the app compute.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.ComputeState"
                     }
                   },
@@ -3701,7 +3709,7 @@
                   "description": "Write-only setting. Specifies the user or service principal that the job runs as. If not specified, the job runs as the user who created the job.\n\nEither `user_name` or `service_principal_name` should be specified. If not, an error is thrown.",
                   "properties": {
                     "service_principal_name": {
-                      "description": "Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.",
+                      "description": "The application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.",
                       "$ref": "#/$defs/string"
                     },
                     "user_name": {
@@ -7227,12 +7235,13 @@
   "properties": {
     "artifacts": {
       "description": "Defines the attributes to build an artifact",
-      "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact"
+      "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact",
+      "markdownDescription": "Defines the attributes to build artifacts, where each key is the name of the artifact, and the value is a Map that defines the artifact build settings. For information about the `artifacts` mapping, see [artifacts](https://docs.databricks.com/dev-tools/bundles/settings.html#artifacts).\n\nArtifact settings defined in the top level of the bundle configuration can be overridden in the `targets` mapping. See [link](https://docs.databricks.com/dev-tools/bundles/artifact-overrides.html)."
     },
     "bundle": {
-      "description": "The attributes of the bundle.",
+      "description": "The bundle attributes when deploying to this target.",
       "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Bundle",
-      "markdownDescription": "The attributes of the bundle. See [bundle](https://docs.databricks.com/dev-tools/bundles/settings.html#bundle)"
+      "markdownDescription": "The bundle attributes when deploying to this target,"
     },
     "experimental": {
       "description": "Defines attributes for experimental features.",
@@ -7244,9 +7253,9 @@
       "markdownDescription": "Specifies a list of path globs that contain configuration files to include within the bundle. See [include](https://docs.databricks.com/dev-tools/bundles/settings.html#include)"
     },
     "permissions": {
-      "description": "Defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle",
+      "description": "Defines a permission for a specific entity.",
       "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission",
-      "markdownDescription": "Defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle. See [permissions](https://docs.databricks.com/dev-tools/bundles/settings.html#permissions) and [link](https://docs.databricks.com/dev-tools/bundles/permissions.html)."
+      "markdownDescription": "A Sequence that defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle, where each item in the sequence is a permission for a specific entity.\n\nSee [permissions](https://docs.databricks.com/dev-tools/bundles/settings.html#permissions) and [link](https://docs.databricks.com/dev-tools/bundles/permissions.html)."
     },
     "presets": {
       "description": "Defines bundle deployment presets.",
@@ -7254,22 +7263,24 @@
       "markdownDescription": "Defines bundle deployment presets. See [presets](https://docs.databricks.com/dev-tools/bundles/deployment-modes.html#presets)."
     },
     "resources": {
-      "description": "Specifies information about the Databricks resources used by the bundle",
+      "description": "A Map that defines the resources for the bundle, where each key is the name of the resource, and the value is a Map that defines the resource.",
       "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Resources",
-      "markdownDescription": "Specifies information about the Databricks resources used by the bundle. See [link](https://docs.databricks.com/dev-tools/bundles/resources.html)."
+      "markdownDescription": "A Map that defines the resources for the bundle, where each key is the name of the resource, and the value is a Map that defines the resource. For more information about \u003cDABS\u003e supported resources, and resource definition reference, see [link](https://docs.databricks.com/dev-tools/bundles/resources.html).\n\n```yaml\nresources:\n  \u003cresource-type\u003e:\n    \u003cresource-name\u003e:\n      \u003cresource-field-name\u003e: \u003cresource-field-value\u003e\n```"
     },
     "run_as": {
-      "description": "The identity to use to run the bundle.",
-      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs"
+      "description": "The identity to use when running \u003cDABS\u003e workflows.",
+      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs",
+      "markdownDescription": "The identity to use when running \u003cDABS\u003e workflows. See [link](https://docs.databricks.com/dev-tools/bundles/run-as.html)."
     },
     "sync": {
       "description": "The files and file paths to include or exclude in the bundle.",
       "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Sync",
-      "markdownDescription": "The files and file paths to include or exclude in the bundle. See [link](https://docs.databricks.com/dev-tools/bundles/)"
+      "markdownDescription": "The files and file paths to include or exclude in the bundle. See [sync](https://docs.databricks.com/dev-tools/bundles/settings.html#sync)."
     },
     "targets": {
       "description": "Defines deployment targets for the bundle.",
-      "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Target"
+      "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Target",
+      "markdownDescription": "Defines deployment targets for the bundle. See [targets](https://docs.databricks.com/dev-tools/bundles/settings.html#targets)"
     },
     "variables": {
       "description": "A Map that defines the custom variables for the bundle, where each key is the name of the variable, and the value is a Map that defines the variable.",
@@ -7277,7 +7288,8 @@
     },
     "workspace": {
       "description": "Defines the Databricks workspace for the bundle.",
-      "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace"
+      "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace",
+      "markdownDescription": "Defines the Databricks workspace for the bundle. See [workspace](https://docs.databricks.com/dev-tools/bundles/settings.html#workspace)."
     }
   },
   "additionalProperties": {}
diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go
index 6f8f39d96..ce25cb023 100644
--- a/libs/jsonschema/from_type.go
+++ b/libs/jsonschema/from_type.go
@@ -111,6 +111,10 @@ func FromType(typ reflect.Type, fns []func(typ reflect.Type, s Schema) Schema) (
 	return res, nil
 }
 
+func TypePath(typ reflect.Type) string {
+	return typePath(typ)
+}
+
 // typePath computes a unique string representation of the type. $ref in the generated
 // JSON schema will refer to this path. See TestTypePath for examples outputs.
 func typePath(typ reflect.Type) string {
diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go
index e63dde359..5028bb0d7 100644
--- a/libs/jsonschema/schema.go
+++ b/libs/jsonschema/schema.go
@@ -76,6 +76,10 @@ type Schema struct {
 	// Title of the object, rendered as inline documentation in the IDE.
 	// https://json-schema.org/understanding-json-schema/reference/annotations
 	Title string `json:"title,omitempty"`
+
+	// Examples of the value for properties in the schema.
+	// https://json-schema.org/understanding-json-schema/reference/annotations
+	Examples []any `json:"examples,omitempty"`
 }
 
 // Default value defined in a JSON Schema, represented as a string.

From 59d6fbfee98bf102716f774de3de99aca129a9c3 Mon Sep 17 00:00:00 2001
From: Ilya Kuznetsov <ilya.kuznetsov@databricks.com>
Date: Wed, 29 Jan 2025 14:34:26 +0100
Subject: [PATCH 169/247] Restore variable file tests (#2220)

## Changes

Uncomment flaky tests, they work properly with latest changes from main

## Tests
<!-- How is this tested? -->
---
 .../bundle/variables/file-defaults/output.txt | 31 +++++++++++++++++++
 .../bundle/variables/file-defaults/script     |  8 ++---
 .../bundle/variables/file-defaults/test.toml  |  8 +++++
 3 files changed, 43 insertions(+), 4 deletions(-)
 create mode 100644 acceptance/bundle/variables/file-defaults/test.toml

diff --git a/acceptance/bundle/variables/file-defaults/output.txt b/acceptance/bundle/variables/file-defaults/output.txt
index 73830aae3..5b01a1b66 100644
--- a/acceptance/bundle/variables/file-defaults/output.txt
+++ b/acceptance/bundle/variables/file-defaults/output.txt
@@ -39,6 +39,37 @@
   }
 }
 
+=== file cannot be parsed
+>>> errcode $CLI bundle validate -o json --target invalid_json
+Error: failed to parse variables file $TMPDIR/.databricks/bundle/invalid_json/variable-overrides.json: error decoding JSON at :0:0: invalid character 'o' in literal false (expecting 'a')
+
+
+Exit code: 1
+{
+  "job_cluster_key": "${var.cluster_key}",
+  "new_cluster": {
+    "node_type_id": "${var.cluster.node_type_id}",
+    "num_workers": "${var.cluster_workers}"
+  }
+}
+
+=== file has wrong structure
+>>> errcode $CLI bundle validate -o json --target wrong_file_structure
+Error: failed to parse variables file $TMPDIR/.databricks/bundle/wrong_file_structure/variable-overrides.json: invalid format
+
+Variables file must be a JSON object with the following format:
+{"var1": "value1", "var2": "value2"}
+
+
+Exit code: 1
+{
+  "job_cluster_key": "${var.cluster_key}",
+  "new_cluster": {
+    "node_type_id": "${var.cluster.node_type_id}",
+    "num_workers": "${var.cluster_workers}"
+  }
+}
+
 === file has variable that is complex but default is string
 >>> errcode $CLI bundle validate -o json --target complex_to_string
 Error: variable cluster_key is not of type complex, but the value in the variable file is a complex type
diff --git a/acceptance/bundle/variables/file-defaults/script b/acceptance/bundle/variables/file-defaults/script
index c5b208755..8e6fd0d75 100644
--- a/acceptance/bundle/variables/file-defaults/script
+++ b/acceptance/bundle/variables/file-defaults/script
@@ -14,11 +14,11 @@ trace BUNDLE_VAR_cluster_key=mlops_stacks-cluster-overriden $CLI bundle validate
 title "variable has value in config file"
 trace $CLI bundle validate -o json --target with_value | jq $cluster_expr
 
-# title "file cannot be parsed"
-# trace errcode $CLI bundle validate -o json --target invalid_json | jq $cluster_expr
+title "file cannot be parsed"
+trace errcode $CLI bundle validate -o json --target invalid_json | jq $cluster_expr
 
-# title "file has wrong structure"
-# trace errcode $CLI bundle validate -o json --target wrong_file_structure | jq $cluster_expr
+title "file has wrong structure"
+trace errcode $CLI bundle validate -o json --target wrong_file_structure | jq $cluster_expr
 
 title "file has variable that is complex but default is string"
 trace errcode $CLI bundle validate -o json --target complex_to_string | jq $cluster_expr
diff --git a/acceptance/bundle/variables/file-defaults/test.toml b/acceptance/bundle/variables/file-defaults/test.toml
new file mode 100644
index 000000000..08403b606
--- /dev/null
+++ b/acceptance/bundle/variables/file-defaults/test.toml
@@ -0,0 +1,8 @@
+# Fix for windows
+[[Repls]]
+Old = '\$TMPDIR\\.databricks\\bundle\\wrong_file_structure\\variable-overrides.json'
+New = '$$TMPDIR/.databricks/bundle/wrong_file_structure/variable-overrides.json'
+
+[[Repls]]
+Old = '\$TMPDIR\\.databricks\\bundle\\invalid_json\\variable-overrides.json'
+New = '$$TMPDIR/.databricks/bundle/invalid_json/variable-overrides.json'

From ec7808da34cb93c5a4c220edd1dc70b6e40ca4af Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Wed, 29 Jan 2025 14:38:28 +0100
Subject: [PATCH 170/247] Added support for double underscore variable
 references (#2203)

## Changes
Added support for double underscore variable references.

Previously we made this restriction stronger with no particular reason,
TF provider supports multiple underscores and thus DABs should do as
well.

Fixes #1753

## Tests
Added acceptance and integration tests
---
 .../double_underscore/databricks.yml          | 14 +++++++
 .../variables/double_underscore/output.txt    |  7 ++++
 .../bundle/variables/double_underscore/script |  1 +
 integration/bundle/basic_test.go              | 39 +++++++++++++++++++
 .../databricks_template_schema.json           | 21 ++++++++++
 .../template/databricks.yml.tmpl              | 32 +++++++++++++++
 .../template/hello_world.py                   |  1 +
 .../basic_with_variables/bundle_deploy.txt    |  4 ++
 .../basic_with_variables/bundle_validate.txt  |  7 ++++
 libs/dyn/dynvar/ref.go                        |  6 ++-
 libs/dyn/dynvar/ref_test.go                   | 12 +++---
 11 files changed, 138 insertions(+), 6 deletions(-)
 create mode 100644 acceptance/bundle/variables/double_underscore/databricks.yml
 create mode 100644 acceptance/bundle/variables/double_underscore/output.txt
 create mode 100644 acceptance/bundle/variables/double_underscore/script
 create mode 100644 integration/bundle/bundles/basic_with_variables/databricks_template_schema.json
 create mode 100644 integration/bundle/bundles/basic_with_variables/template/databricks.yml.tmpl
 create mode 100644 integration/bundle/bundles/basic_with_variables/template/hello_world.py
 create mode 100644 integration/bundle/testdata/basic_with_variables/bundle_deploy.txt
 create mode 100644 integration/bundle/testdata/basic_with_variables/bundle_validate.txt

diff --git a/acceptance/bundle/variables/double_underscore/databricks.yml b/acceptance/bundle/variables/double_underscore/databricks.yml
new file mode 100644
index 000000000..3bb15d42d
--- /dev/null
+++ b/acceptance/bundle/variables/double_underscore/databricks.yml
@@ -0,0 +1,14 @@
+bundle:
+  name: double_underscore
+
+variables:
+  double__underscore:
+    description: "This is a variable with a double underscore"
+    default: "default"
+
+resources:
+  jobs:
+    test_job:
+      name: "test"
+      tasks:
+        - task_key: "test ${var.double__underscore}"
diff --git a/acceptance/bundle/variables/double_underscore/output.txt b/acceptance/bundle/variables/double_underscore/output.txt
new file mode 100644
index 000000000..45529038d
--- /dev/null
+++ b/acceptance/bundle/variables/double_underscore/output.txt
@@ -0,0 +1,7 @@
+
+>>> $CLI bundle validate -o json
+[
+  {
+    "task_key": "test default"
+  }
+]
diff --git a/acceptance/bundle/variables/double_underscore/script b/acceptance/bundle/variables/double_underscore/script
new file mode 100644
index 000000000..a7394df77
--- /dev/null
+++ b/acceptance/bundle/variables/double_underscore/script
@@ -0,0 +1 @@
+trace $CLI bundle validate -o json | jq .resources.jobs.test_job.tasks
diff --git a/integration/bundle/basic_test.go b/integration/bundle/basic_test.go
index 79301b850..53f8e3ef6 100644
--- a/integration/bundle/basic_test.go
+++ b/integration/bundle/basic_test.go
@@ -6,7 +6,9 @@ import (
 	"testing"
 
 	"github.com/databricks/cli/integration/internal/acc"
+	"github.com/databricks/cli/internal/testcli"
 	"github.com/databricks/cli/internal/testutil"
+	"github.com/databricks/cli/libs/testdiff"
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 )
@@ -35,3 +37,40 @@ func TestBasicBundleDeployWithFailOnActiveRuns(t *testing.T) {
 	// deploy empty bundle again
 	deployBundleWithFlags(t, ctx, root, []string{"--fail-on-active-runs"})
 }
+
+func TestBasicBundleDeployWithDoubleUnderscoreVariables(t *testing.T) {
+	ctx, wt := acc.WorkspaceTest(t)
+
+	nodeTypeId := testutil.GetCloud(t).NodeTypeID()
+	uniqueId := uuid.New().String()
+	root := initTestTemplate(t, ctx, "basic_with_variables", map[string]any{
+		"unique_id":     uniqueId,
+		"node_type_id":  nodeTypeId,
+		"spark_version": defaultSparkVersion,
+	})
+
+	currentUser, err := wt.W.CurrentUser.Me(ctx)
+	require.NoError(t, err)
+
+	ctx, replacements := testdiff.WithReplacementsMap(ctx)
+	replacements.Set(uniqueId, "$UNIQUE_PRJ")
+	replacements.Set(currentUser.UserName, "$USERNAME")
+
+	t.Cleanup(func() {
+		destroyBundle(t, ctx, root)
+	})
+
+	testutil.Chdir(t, root)
+	testcli.AssertOutput(
+		t,
+		ctx,
+		[]string{"bundle", "validate"},
+		testutil.TestData("testdata/basic_with_variables/bundle_validate.txt"),
+	)
+	testcli.AssertOutput(
+		t,
+		ctx,
+		[]string{"bundle", "deploy", "--force-lock", "--auto-approve"},
+		testutil.TestData("testdata/basic_with_variables/bundle_deploy.txt"),
+	)
+}
diff --git a/integration/bundle/bundles/basic_with_variables/databricks_template_schema.json b/integration/bundle/bundles/basic_with_variables/databricks_template_schema.json
new file mode 100644
index 000000000..41a723b0f
--- /dev/null
+++ b/integration/bundle/bundles/basic_with_variables/databricks_template_schema.json
@@ -0,0 +1,21 @@
+{
+    "properties": {
+        "unique_id": {
+            "type": "string",
+            "description": "Unique ID for job name"
+        },
+        "spark_version": {
+            "type": "string",
+            "description": "Spark version used for job cluster"
+        },
+        "node_type_id": {
+            "type": "string",
+            "description": "Node type id for job cluster"
+        },
+        "root_path": {
+            "type": "string",
+            "description": "Root path to deploy bundle to",
+            "default": ""
+        }
+    }
+}
diff --git a/integration/bundle/bundles/basic_with_variables/template/databricks.yml.tmpl b/integration/bundle/bundles/basic_with_variables/template/databricks.yml.tmpl
new file mode 100644
index 000000000..cb02c9e2f
--- /dev/null
+++ b/integration/bundle/bundles/basic_with_variables/template/databricks.yml.tmpl
@@ -0,0 +1,32 @@
+bundle:
+  name: basic
+
+workspace:
+  {{ if .root_path }}
+  root_path: "{{.root_path}}/.bundle/{{.unique_id}}"
+  {{ else }}
+  root_path: "~/.bundle/{{.unique_id}}"
+  {{ end }}
+
+variables:
+  task__key: # Note: the variable has double underscore
+    default: my_notebook_task
+
+resources:
+  jobs:
+    foo__bar: # Note: the resource has double underscore to check that TF provider can use such names
+      name: test-job-basic-{{.unique_id}}
+      tasks:
+        - task_key: ${var.task__key}
+          new_cluster:
+            num_workers: 1
+            spark_version: "{{.spark_version}}"
+            node_type_id: "{{.node_type_id}}"
+          spark_python_task:
+            python_file: ./hello_world.py
+    foo:
+      name: test-job-basic-ref-{{.unique_id}}
+      tasks:
+        - task_key: job_task
+          run_job_task:
+            job_id: ${resources.jobs.foo__bar.id}
diff --git a/integration/bundle/bundles/basic_with_variables/template/hello_world.py b/integration/bundle/bundles/basic_with_variables/template/hello_world.py
new file mode 100644
index 000000000..f301245e2
--- /dev/null
+++ b/integration/bundle/bundles/basic_with_variables/template/hello_world.py
@@ -0,0 +1 @@
+print("Hello World!")
diff --git a/integration/bundle/testdata/basic_with_variables/bundle_deploy.txt b/integration/bundle/testdata/basic_with_variables/bundle_deploy.txt
new file mode 100644
index 000000000..211164174
--- /dev/null
+++ b/integration/bundle/testdata/basic_with_variables/bundle_deploy.txt
@@ -0,0 +1,4 @@
+Uploading bundle files to /Workspace/Users/$USERNAME/.bundle/$UNIQUE_PRJ/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
diff --git a/integration/bundle/testdata/basic_with_variables/bundle_validate.txt b/integration/bundle/testdata/basic_with_variables/bundle_validate.txt
new file mode 100644
index 000000000..dc9016a0f
--- /dev/null
+++ b/integration/bundle/testdata/basic_with_variables/bundle_validate.txt
@@ -0,0 +1,7 @@
+Name: basic
+Target: default
+Workspace:
+  User: $USERNAME
+  Path: /Workspace/Users/$USERNAME/.bundle/$UNIQUE_PRJ
+
+Validation OK!
diff --git a/libs/dyn/dynvar/ref.go b/libs/dyn/dynvar/ref.go
index a28938823..ba397267a 100644
--- a/libs/dyn/dynvar/ref.go
+++ b/libs/dyn/dynvar/ref.go
@@ -1,12 +1,16 @@
 package dynvar
 
 import (
+	"fmt"
 	"regexp"
 
 	"github.com/databricks/cli/libs/dyn"
 )
 
-var re = regexp.MustCompile(`\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`)
+var (
+	baseVarDef = `[a-zA-Z]+([-_]*[a-zA-Z0-9]+)*`
+	re         = regexp.MustCompile(fmt.Sprintf(`\$\{(%s(\.%s(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`, baseVarDef, baseVarDef))
+)
 
 // ref represents a variable reference.
 // It is a string [dyn.Value] contained in a larger [dyn.Value].
diff --git a/libs/dyn/dynvar/ref_test.go b/libs/dyn/dynvar/ref_test.go
index 4110732f8..637ecb98e 100644
--- a/libs/dyn/dynvar/ref_test.go
+++ b/libs/dyn/dynvar/ref_test.go
@@ -15,9 +15,13 @@ func TestNewRefNoString(t *testing.T) {
 
 func TestNewRefValidPattern(t *testing.T) {
 	for in, refs := range map[string][]string{
-		"${hello_world.world_world}": {"hello_world.world_world"},
-		"${helloworld.world-world}":  {"helloworld.world-world"},
-		"${hello-world.world-world}": {"hello-world.world-world"},
+		"${hello_world.world_world}":  {"hello_world.world_world"},
+		"${helloworld.world-world}":   {"helloworld.world-world"},
+		"${hello-world.world-world}":  {"hello-world.world-world"},
+		"${hello_world.world__world}": {"hello_world.world__world"},
+		"${hello_world.world--world}": {"hello_world.world--world"},
+		"${hello_world.world-_world}": {"hello_world.world-_world"},
+		"${hello_world.world_-world}": {"hello_world.world_-world"},
 	} {
 		ref, ok := newRef(dyn.V(in))
 		require.True(t, ok, "should match valid pattern: %s", in)
@@ -36,8 +40,6 @@ func TestNewRefInvalidPattern(t *testing.T) {
 		"${_-_._-_.id}",                 // cannot use _- in sequence
 		"${0helloworld.world-world}",    // interpolated first section shouldn't start with number
 		"${helloworld.9world-world}",    // interpolated second section shouldn't start with number
-		"${a-a.a-_a-a.id}",              // fails because of -_ in the second segment
-		"${a-a.a--a-a.id}",              // fails because of -- in the second segment
 	}
 	for _, v := range invalid {
 		_, ok := newRef(dyn.V(v))

From 13596eb6052fc90a4df1ab3a0130a0a568959c18 Mon Sep 17 00:00:00 2001
From: Gleb Kanterov <kanterov@users.noreply.github.com>
Date: Wed, 29 Jan 2025 14:56:57 +0100
Subject: [PATCH 171/247] PythonMutator: Fix relative path error (#2253)

## Changes

Fix relative path errors in the Python mutator that was failing during
deployment since v0.239.1.

Before that:

```
% databricks bundle deploy
Deploying resources...
Updating deployment state...
Error: failed to compute relative path for job jobs_as_code_project_job: Rel: can't make resources/jobs_as_code_project_job.py relative to /Users/$USER/jobs_as_code_project
```

As a result, the bundle was deployed, but the deployment state wasn't
updated.

## Tests

Unit tests, adding acceptance tests in
https://github.com/databricks/cli/pull/2254
---
 .../config/mutator/python/python_locations.go |  9 ++++++-
 .../mutator/python/python_locations_test.go   | 24 +++++++++++++++----
 .../config/mutator/python/python_mutator.go   |  6 ++---
 .../mutator/python/python_mutator_test.go     |  7 +++++-
 4 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/bundle/config/mutator/python/python_locations.go b/bundle/config/mutator/python/python_locations.go
index 2fa86bea0..9cb65c302 100644
--- a/bundle/config/mutator/python/python_locations.go
+++ b/bundle/config/mutator/python/python_locations.go
@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	pathlib "path"
 	"path/filepath"
 
 	"github.com/databricks/cli/libs/dyn"
@@ -99,7 +100,7 @@ func removeVirtualLocations(locations []dyn.Location) []dyn.Location {
 // parsePythonLocations parses locations.json from the Python mutator.
 //
 // locations file is newline-separated JSON objects with pythonLocationEntry structure.
-func parsePythonLocations(input io.Reader) (*pythonLocations, error) {
+func parsePythonLocations(bundleRoot string, input io.Reader) (*pythonLocations, error) {
 	decoder := json.NewDecoder(input)
 	locations := newPythonLocations()
 
@@ -116,6 +117,12 @@ func parsePythonLocations(input io.Reader) (*pythonLocations, error) {
 			return nil, fmt.Errorf("failed to parse python location: %s", err)
 		}
 
+		// Output can contain both relative paths and absolute paths outside of bundle root.
+		// Mutator pipeline expects all path to be absolute at this point, so make all paths absolute.
+		if !pathlib.IsAbs(entry.File) {
+			entry.File = filepath.Join(bundleRoot, entry.File)
+		}
+
 		location := dyn.Location{
 			File:   entry.File,
 			Line:   entry.Line,
diff --git a/bundle/config/mutator/python/python_locations_test.go b/bundle/config/mutator/python/python_locations_test.go
index 32afcc92b..2860af820 100644
--- a/bundle/config/mutator/python/python_locations_test.go
+++ b/bundle/config/mutator/python/python_locations_test.go
@@ -165,12 +165,28 @@ func TestLoadOutput(t *testing.T) {
 	require.Equal(t, filepath.Join(bundleRoot, generatedFileName), notebookPath.Locations()[0].File)
 }
 
-func TestParsePythonLocations(t *testing.T) {
-	expected := dyn.Location{File: "foo.py", Line: 1, Column: 2}
+func TestParsePythonLocations_absolutePath(t *testing.T) {
+	// output can contain absolute path that is outside of the bundle root
+	expected := dyn.Location{File: "/Shared/foo.py", Line: 1, Column: 2}
 
-	input := `{"path": "foo", "file": "foo.py", "line": 1, "column": 2}`
+	input := `{"path": "foo", "file": "/Shared/foo.py", "line": 1, "column": 2}`
 	reader := bytes.NewReader([]byte(input))
-	locations, err := parsePythonLocations(reader)
+	locations, err := parsePythonLocations("/tmp/", reader)
+
+	assert.NoError(t, err)
+
+	assert.True(t, locations.keys["foo"].exists)
+	assert.Equal(t, expected, locations.keys["foo"].location)
+}
+
+func TestParsePythonLocations_relativePath(t *testing.T) {
+	// output can contain relative paths, we expect all locations to be absolute
+	// at this stage of mutator pipeline
+	expected := dyn.Location{File: filepath.Clean("/tmp/my_project/foo.py"), Line: 1, Column: 2}
+
+	input := `{"path": "foo", "file": "foo.py", "line": 1, "column": 2}`
+	reader := bytes.NewReader([]byte(input))
+	locations, err := parsePythonLocations(filepath.Clean("/tmp/my_project"), reader)
 
 	assert.NoError(t, err)
 
diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go
index cd2e286e5..f75f111cf 100644
--- a/bundle/config/mutator/python/python_mutator.go
+++ b/bundle/config/mutator/python/python_mutator.go
@@ -331,7 +331,7 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, root dyn.Value, op
 		return dyn.InvalidValue, diag.Errorf("failed to load diagnostics: %s", pythonDiagnosticsErr)
 	}
 
-	locations, err := loadLocationsFile(locationsPath)
+	locations, err := loadLocationsFile(opts.bundleRootPath, locationsPath)
 	if err != nil {
 		return dyn.InvalidValue, diag.Errorf("failed to load locations: %s", err)
 	}
@@ -381,7 +381,7 @@ func writeInputFile(inputPath string, input dyn.Value) error {
 }
 
 // loadLocationsFile loads locations.json containing source locations for generated YAML.
-func loadLocationsFile(locationsPath string) (*pythonLocations, error) {
+func loadLocationsFile(bundleRoot, locationsPath string) (*pythonLocations, error) {
 	locationsFile, err := os.Open(locationsPath)
 	if errors.Is(err, fs.ErrNotExist) {
 		return newPythonLocations(), nil
@@ -391,7 +391,7 @@ func loadLocationsFile(locationsPath string) (*pythonLocations, error) {
 
 	defer locationsFile.Close()
 
-	return parsePythonLocations(locationsFile)
+	return parsePythonLocations(bundleRoot, locationsFile)
 }
 
 func loadOutputFile(rootPath, outputPath string, locations *pythonLocations) (dyn.Value, diag.Diagnostics) {
diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go
index 322fb79e8..9d957e797 100644
--- a/bundle/config/mutator/python/python_mutator_test.go
+++ b/bundle/config/mutator/python/python_mutator_test.go
@@ -54,6 +54,8 @@ func TestPythonMutator_Name_applyMutators(t *testing.T) {
 func TestPythonMutator_loadResources(t *testing.T) {
 	withFakeVEnv(t, ".venv")
 
+	rootPath := filepath.Join(t.TempDir(), "my_project")
+
 	b := loadYaml("databricks.yml", `
       experimental:
         python:
@@ -64,6 +66,9 @@ func TestPythonMutator_loadResources(t *testing.T) {
           job0:
             name: job_0`)
 
+	// set rootPath so that we can make absolute paths in dyn.Location
+	b.BundleRootPath = rootPath
+
 	ctx := withProcessStub(
 		t,
 		[]string{
@@ -120,7 +125,7 @@ func TestPythonMutator_loadResources(t *testing.T) {
 
 		assert.Equal(t, []dyn.Location{
 			{
-				File:   "src/examples/job1.py",
+				File:   filepath.Join(rootPath, "src/examples/job1.py"),
 				Line:   5,
 				Column: 7,
 			},

From c3a6e11627e1cb0b10c2289fbfe39a2a36fa641a Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Wed, 29 Jan 2025 19:35:58 +0530
Subject: [PATCH 172/247] Add integration test for the /telemetry-ext endpoint
 (#2259)

## Changes
Followup from
https://github.com/databricks/cli/pull/2209#pullrequestreview-2580308075.

This PR adds an integration test to validate that the API type bindings
work against the telemetry endpoint.

## Tests
N/A

---------

Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>
---
 integration/libs/telemetry/telemetry_test.go | 65 ++++++++++++++++++++
 1 file changed, 65 insertions(+)
 create mode 100644 integration/libs/telemetry/telemetry_test.go

diff --git a/integration/libs/telemetry/telemetry_test.go b/integration/libs/telemetry/telemetry_test.go
new file mode 100644
index 000000000..d329c238e
--- /dev/null
+++ b/integration/libs/telemetry/telemetry_test.go
@@ -0,0 +1,65 @@
+package telemetry
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/databricks/cli/integration/internal/acc"
+	"github.com/databricks/cli/libs/telemetry"
+	"github.com/databricks/cli/libs/telemetry/protos"
+	"github.com/databricks/databricks-sdk-go/client"
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTelemetryEndpoint(t *testing.T) {
+	ctx, wt := acc.WorkspaceTest(t)
+	w := wt.W
+
+	apiClient, err := client.New(w.Config)
+	require.NoError(t, err)
+
+	logs := []protos.FrontendLog{
+		{
+			FrontendLogEventID: uuid.New().String(),
+			Entry: protos.FrontendLogEntry{
+				DatabricksCliLog: protos.DatabricksCliLog{
+					CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue1},
+				},
+			},
+		},
+		{
+			FrontendLogEventID: uuid.New().String(),
+			Entry: protos.FrontendLogEntry{
+				DatabricksCliLog: protos.DatabricksCliLog{
+					CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue2},
+				},
+			},
+		},
+	}
+
+	protoLogs := make([]string, len(logs))
+	for i, log := range logs {
+		b, err := json.Marshal(log)
+		require.NoError(t, err)
+		protoLogs[i] = string(b)
+	}
+
+	reqB := telemetry.RequestBody{
+		UploadTime: time.Now().UnixMilli(),
+		Items:      []string{},
+		ProtoLogs:  protoLogs,
+	}
+
+	respB := telemetry.ResponseBody{}
+
+	err = apiClient.Do(ctx, "POST", "/telemetry-ext", nil, nil, reqB, &respB)
+	require.NoError(t, err)
+
+	assert.Equal(t, telemetry.ResponseBody{
+		Errors:          []telemetry.LogError{},
+		NumProtoSuccess: int64(2),
+	}, respB)
+}

From 38efedcd736fad3af8f985fd9cab26f246d69dfd Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 29 Jan 2025 15:15:52 +0100
Subject: [PATCH 173/247] Remove bundle.git.inferred (#2258)

The only use case for it was to emit a warning and based on the
discussion here
https://github.com/databricks/cli/pull/2213/files#r1933558087 the
warning it not useful and logging that with reduced severity is also not
useful.
---
 acceptance/bundle/git-permerror/output.txt     | 18 ++++++------------
 .../variables/prepend-workspace-var/output.txt |  3 +--
 bundle/config/git.go                           |  3 ---
 bundle/config/mutator/load_git_details.go      |  1 -
 bundle/config/mutator/process_target_mode.go   |  5 -----
 bundle/deploy/metadata/compute_test.go         |  4 ----
 bundle/tests/environment_git_test.go           |  2 --
 bundle/tests/git_test.go                       |  3 ---
 .../testdata/default_python/bundle_summary.txt |  3 +--
 9 files changed, 8 insertions(+), 34 deletions(-)

diff --git a/acceptance/bundle/git-permerror/output.txt b/acceptance/bundle/git-permerror/output.txt
index 2b52134ab..60e77ca0e 100644
--- a/acceptance/bundle/git-permerror/output.txt
+++ b/acceptance/bundle/git-permerror/output.txt
@@ -21,8 +21,7 @@ Error: unable to load repository specific gitconfig: open config: permission den
 
 Exit code: 1
 {
-  "bundle_root_path": ".",
-  "inferred": true
+  "bundle_root_path": "."
 }
 
 >>> withdir subdir/a/b $CLI bundle validate -o json
@@ -31,8 +30,7 @@ Error: unable to load repository specific gitconfig: open config: permission den
 
 Exit code: 1
 {
-  "bundle_root_path": ".",
-  "inferred": true
+  "bundle_root_path": "."
 }
 
 
@@ -42,14 +40,12 @@ Exit code: 1
 
 >>> $CLI bundle validate -o json
 {
-  "bundle_root_path": ".",
-  "inferred": true
+  "bundle_root_path": "."
 }
 
 >>> withdir subdir/a/b $CLI bundle validate -o json
 {
-  "bundle_root_path": ".",
-  "inferred": true
+  "bundle_root_path": "."
 }
 
 
@@ -63,8 +59,7 @@ Error: unable to load repository specific gitconfig: open config: permission den
 
 Exit code: 1
 {
-  "bundle_root_path": ".",
-  "inferred": true
+  "bundle_root_path": "."
 }
 
 >>> withdir subdir/a/b $CLI bundle validate -o json
@@ -73,6 +68,5 @@ Error: unable to load repository specific gitconfig: open config: permission den
 
 Exit code: 1
 {
-  "bundle_root_path": ".",
-  "inferred": true
+  "bundle_root_path": "."
 }
diff --git a/acceptance/bundle/variables/prepend-workspace-var/output.txt b/acceptance/bundle/variables/prepend-workspace-var/output.txt
index 575fac6d4..706d134ff 100644
--- a/acceptance/bundle/variables/prepend-workspace-var/output.txt
+++ b/acceptance/bundle/variables/prepend-workspace-var/output.txt
@@ -3,8 +3,7 @@
   "bundle": {
     "environment": "dev",
     "git": {
-      "bundle_root_path": ".",
-      "inferred": true
+      "bundle_root_path": "."
     },
     "target": "dev",
     "terraform": {
diff --git a/bundle/config/git.go b/bundle/config/git.go
index f9f2f83e5..4b89bc2d2 100644
--- a/bundle/config/git.go
+++ b/bundle/config/git.go
@@ -8,9 +8,6 @@ type Git struct {
 	// Path to bundle root relative to the git repository root.
 	BundleRootPath string `json:"bundle_root_path,omitempty" bundle:"readonly"`
 
-	// Inferred is set to true if the Git details were inferred and weren't set explicitly
-	Inferred bool `json:"inferred,omitempty" bundle:"readonly"`
-
 	// The actual branch according to Git (may be different from the configured branch)
 	ActualBranch string `json:"actual_branch,omitempty" bundle:"readonly"`
 }
diff --git a/bundle/config/mutator/load_git_details.go b/bundle/config/mutator/load_git_details.go
index 3661c6bcd..dea948fcb 100644
--- a/bundle/config/mutator/load_git_details.go
+++ b/bundle/config/mutator/load_git_details.go
@@ -40,7 +40,6 @@ func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn
 	b.Config.Bundle.Git.ActualBranch = info.CurrentBranch
 	if b.Config.Bundle.Git.Branch == "" {
 		// Only load branch if there's no user defined value
-		b.Config.Bundle.Git.Inferred = true
 		b.Config.Bundle.Git.Branch = info.CurrentBranch
 	}
 
diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go
index 0fe6bd54f..576f0c352 100644
--- a/bundle/config/mutator/process_target_mode.go
+++ b/bundle/config/mutator/process_target_mode.go
@@ -135,11 +135,6 @@ func findNonUserPath(b *bundle.Bundle) string {
 }
 
 func validateProductionMode(ctx context.Context, b *bundle.Bundle, isPrincipalUsed bool) diag.Diagnostics {
-	if b.Config.Bundle.Git.Inferred {
-		env := b.Config.Bundle.Target
-		log.Warnf(ctx, "target with 'mode: production' should specify an explicit 'targets.%s.git' configuration", env)
-	}
-
 	r := b.Config.Resources
 	for i := range r.Pipelines {
 		if r.Pipelines[i].Development {
diff --git a/bundle/deploy/metadata/compute_test.go b/bundle/deploy/metadata/compute_test.go
index c6fa9bddb..64f899695 100644
--- a/bundle/deploy/metadata/compute_test.go
+++ b/bundle/deploy/metadata/compute_test.go
@@ -31,7 +31,6 @@ func TestComputeMetadataMutator(t *testing.T) {
 					OriginURL:      "www.host.com",
 					Commit:         "abcd",
 					BundleRootPath: "a/b/c/d",
-					Inferred:       true,
 				},
 			},
 			Resources: config.Resources{
@@ -72,9 +71,6 @@ func TestComputeMetadataMutator(t *testing.T) {
 					OriginURL:      "www.host.com",
 					Commit:         "abcd",
 					BundleRootPath: "a/b/c/d",
-
-					// Test that this field doesn't carry over into the metadata.
-					Inferred: false,
 				},
 			},
 			Resources: metadata.Resources{
diff --git a/bundle/tests/environment_git_test.go b/bundle/tests/environment_git_test.go
index 848b972b1..901d2867b 100644
--- a/bundle/tests/environment_git_test.go
+++ b/bundle/tests/environment_git_test.go
@@ -13,7 +13,6 @@ import (
 func TestGitAutoLoadWithEnvironment(t *testing.T) {
 	b := load(t, "./environments_autoload_git")
 	bundle.Apply(context.Background(), b, mutator.LoadGitDetails())
-	assert.True(t, b.Config.Bundle.Git.Inferred)
 	validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
 	assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)
 }
@@ -21,7 +20,6 @@ func TestGitAutoLoadWithEnvironment(t *testing.T) {
 func TestGitManuallySetBranchWithEnvironment(t *testing.T) {
 	b := loadTarget(t, "./environments_autoload_git", "production")
 	bundle.Apply(context.Background(), b, mutator.LoadGitDetails())
-	assert.False(t, b.Config.Bundle.Git.Inferred)
 	assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
 	validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
 	assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)
diff --git a/bundle/tests/git_test.go b/bundle/tests/git_test.go
index 41293e450..dd79e26a4 100644
--- a/bundle/tests/git_test.go
+++ b/bundle/tests/git_test.go
@@ -14,7 +14,6 @@ import (
 func TestGitAutoLoad(t *testing.T) {
 	b := load(t, "./autoload_git")
 	bundle.Apply(context.Background(), b, mutator.LoadGitDetails())
-	assert.True(t, b.Config.Bundle.Git.Inferred)
 	validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
 	assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)
 }
@@ -22,7 +21,6 @@ func TestGitAutoLoad(t *testing.T) {
 func TestGitManuallySetBranch(t *testing.T) {
 	b := loadTarget(t, "./autoload_git", "production")
 	bundle.Apply(context.Background(), b, mutator.LoadGitDetails())
-	assert.False(t, b.Config.Bundle.Git.Inferred)
 	assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
 	validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
 	assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)
@@ -36,7 +34,6 @@ func TestGitBundleBranchValidation(t *testing.T) {
 
 	b := load(t, "./git_branch_validation")
 	bundle.Apply(context.Background(), b, mutator.LoadGitDetails())
-	assert.False(t, b.Config.Bundle.Git.Inferred)
 	assert.Equal(t, "feature-a", b.Config.Bundle.Git.Branch)
 	assert.Equal(t, "feature-b", b.Config.Bundle.Git.ActualBranch)
 
diff --git a/integration/bundle/testdata/default_python/bundle_summary.txt b/integration/bundle/testdata/default_python/bundle_summary.txt
index 88ccdc496..0b4c15764 100644
--- a/integration/bundle/testdata/default_python/bundle_summary.txt
+++ b/integration/bundle/testdata/default_python/bundle_summary.txt
@@ -7,8 +7,7 @@
       "exec_path": "/tmp/.../terraform"
     },
     "git": {
-      "bundle_root_path": ".",
-      "inferred": true
+      "bundle_root_path": "."
     },
     "mode": "development",
     "deployment": {

From ce965b22b2dc194ce90d101bf52f892d18e3773d Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Wed, 29 Jan 2025 16:55:53 +0100
Subject: [PATCH 174/247] [Release] Release v0.240.0 (#2264)

Bundles:
* Added support for double underscore variable references
([#2203](https://github.com/databricks/cli/pull/2203)).
* Do not wait for app compute to start on `bundle deploy`
([#2144](https://github.com/databricks/cli/pull/2144)).
* Remove bundle.git.inferred
([#2258](https://github.com/databricks/cli/pull/2258)).
* libs/python: Remove DetectInterpreters
([#2234](https://github.com/databricks/cli/pull/2234)).

API Changes:
 * Added `databricks access-control` command group.
 * Added `databricks serving-endpoints http-request` command.
* Changed `databricks serving-endpoints create` command with new
required argument order.
* Changed `databricks serving-endpoints get-open-api` command return
type to become non-empty.
* Changed `databricks recipients update` command return type to become
non-empty.

OpenAPI commit 0be1b914249781b5e903b7676fd02255755bc851 (2025-01-22)
Dependency updates:
* Bump github.com/databricks/databricks-sdk-go from 0.55.0 to 0.56.1
([#2238](https://github.com/databricks/cli/pull/2238)).
* Upgrade TF provider to 1.64.1
([#2247](https://github.com/databricks/cli/pull/2247)).
---
 CHANGELOG.md | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 255bfb0a8..449c30288 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,25 @@
 # Version changelog
 
+## [Release] Release v0.240.0
+
+Bundles:
+ * Added support for double underscore variable references ([#2203](https://github.com/databricks/cli/pull/2203)).
+ * Do not wait for app compute to start on `bundle deploy` ([#2144](https://github.com/databricks/cli/pull/2144)).
+ * Remove bundle.git.inferred ([#2258](https://github.com/databricks/cli/pull/2258)).
+ * libs/python: Remove DetectInterpreters ([#2234](https://github.com/databricks/cli/pull/2234)).
+
+API Changes:
+ * Added `databricks access-control` command group.
+ * Added `databricks serving-endpoints http-request` command.
+ * Changed `databricks serving-endpoints create` command with new required argument order.
+ * Changed `databricks serving-endpoints get-open-api` command return type to become non-empty.
+ * Changed `databricks recipients update` command return type to become non-empty.
+
+OpenAPI commit 0be1b914249781b5e903b7676fd02255755bc851 (2025-01-22)
+Dependency updates:
+ * Bump github.com/databricks/databricks-sdk-go from 0.55.0 to 0.56.1 ([#2238](https://github.com/databricks/cli/pull/2238)).
+ * Upgrade TF provider to 1.64.1 ([#2247](https://github.com/databricks/cli/pull/2247)).
+
 ## [Release] Release v0.239.1
 
 CLI:

From 55c03cc119c2792b084928f12a15b93b3a6ed14a Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Wed, 29 Jan 2025 21:24:33 +0530
Subject: [PATCH 175/247] Always close test HTTP server during cleanup (#2261)

## Changes
This PR registers the `server.Close()` function to be run during test
cleanup in the server initialization function. This ensures that all
test servers are closed as soon as the test they are scoped to finish.

Motivated by https://github.com/databricks/cli/pull/2255/files where a
regression was introduced where we did not close the test server.

## Tests
N/A
---
 acceptance/cmd_server_test.go | 3 ++-
 acceptance/server_test.go     | 9 ---------
 libs/testserver/server.go     | 5 +----
 3 files changed, 3 insertions(+), 14 deletions(-)

diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go
index 3f5a6356e..9af63d0db 100644
--- a/acceptance/cmd_server_test.go
+++ b/acceptance/cmd_server_test.go
@@ -13,7 +13,8 @@ import (
 )
 
 func StartCmdServer(t *testing.T) *testserver.Server {
-	server := StartServer(t)
+	server := testserver.New(t)
+
 	server.Handle("/", func(r *http.Request) (any, error) {
 		q := r.URL.Query()
 		args := strings.Split(q.Get("args"), " ")
diff --git a/acceptance/server_test.go b/acceptance/server_test.go
index 66de5dcbf..98e351739 100644
--- a/acceptance/server_test.go
+++ b/acceptance/server_test.go
@@ -2,7 +2,6 @@ package acceptance_test
 
 import (
 	"net/http"
-	"testing"
 
 	"github.com/databricks/cli/libs/testserver"
 	"github.com/databricks/databricks-sdk-go/service/catalog"
@@ -11,14 +10,6 @@ import (
 	"github.com/databricks/databricks-sdk-go/service/workspace"
 )
 
-func StartServer(t *testing.T) *testserver.Server {
-	server := testserver.New(t)
-	t.Cleanup(func() {
-		server.Close()
-	})
-	return server
-}
-
 func AddHandlers(server *testserver.Server) {
 	server.Handle("GET /api/2.0/policies/clusters/list", func(r *http.Request) (any, error) {
 		return compute.ListPoliciesResponse{
diff --git a/libs/testserver/server.go b/libs/testserver/server.go
index 10269af8f..9ebfe3ba0 100644
--- a/libs/testserver/server.go
+++ b/libs/testserver/server.go
@@ -18,6 +18,7 @@ type Server struct {
 func New(t testutil.TestingT) *Server {
 	mux := http.NewServeMux()
 	server := httptest.NewServer(mux)
+	t.Cleanup(server.Close)
 
 	return &Server{
 		Server: server,
@@ -28,10 +29,6 @@ func New(t testutil.TestingT) *Server {
 
 type HandlerFunc func(req *http.Request) (resp any, err error)
 
-func (s *Server) Close() {
-	s.Server.Close()
-}
-
 func (s *Server) Handle(pattern string, handler HandlerFunc) {
 	s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) {
 		resp, err := handler(r)

From 58ef34f320f4865d2a62d08b80798fdbdcf7294d Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 29 Jan 2025 18:35:03 +0100
Subject: [PATCH 176/247] acc: Include "id" into /api/2.0/preview/scim/v2/Me
 response (#2266)

This is something terraform provider expects.

Related to https://github.com/databricks/cli/pull/2242
---
 acceptance/bundle/variables/git-branch/output.txt               | 2 ++
 acceptance/bundle/variables/prepend-workspace-var/output.txt    | 1 +
 acceptance/bundle/variables/resolve-builtin/output.txt          | 1 +
 .../bundle/variables/resolve-vars-in-root-path/output.txt       | 1 +
 acceptance/server_test.go                                       | 1 +
 5 files changed, 6 insertions(+)

diff --git a/acceptance/bundle/variables/git-branch/output.txt b/acceptance/bundle/variables/git-branch/output.txt
index d6d824394..fb3ab805a 100644
--- a/acceptance/bundle/variables/git-branch/output.txt
+++ b/acceptance/bundle/variables/git-branch/output.txt
@@ -29,6 +29,7 @@
   "workspace": {
     "artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/artifacts",
     "current_user": {
+      "id": "$USER.Id",
       "short_name": "$USERNAME",
       "userName": "$USERNAME"
     },
@@ -78,6 +79,7 @@ Validation OK!
   "workspace": {
     "artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/artifacts",
     "current_user": {
+      "id": "$USER.Id",
       "short_name": "$USERNAME",
       "userName": "$USERNAME"
     },
diff --git a/acceptance/bundle/variables/prepend-workspace-var/output.txt b/acceptance/bundle/variables/prepend-workspace-var/output.txt
index 706d134ff..fcaa25b4a 100644
--- a/acceptance/bundle/variables/prepend-workspace-var/output.txt
+++ b/acceptance/bundle/variables/prepend-workspace-var/output.txt
@@ -54,6 +54,7 @@
   "workspace": {
     "artifact_path": "/Users/$USERNAME/path/to/root/artifacts",
     "current_user": {
+      "id": "$USER.Id",
       "short_name": "$USERNAME",
       "userName": "$USERNAME"
     },
diff --git a/acceptance/bundle/variables/resolve-builtin/output.txt b/acceptance/bundle/variables/resolve-builtin/output.txt
index f060c472e..0c1678f84 100644
--- a/acceptance/bundle/variables/resolve-builtin/output.txt
+++ b/acceptance/bundle/variables/resolve-builtin/output.txt
@@ -1,6 +1,7 @@
 {
   "artifact_path": "TestResolveVariableReferences/bar/artifacts",
   "current_user": {
+    "id": "$USER.Id",
     "short_name": "$USERNAME",
     "userName": "$USERNAME"
   },
diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt b/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt
index c56fbe415..51eb40c91 100644
--- a/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt
+++ b/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt
@@ -1,6 +1,7 @@
 {
   "artifact_path": "TestResolveVariableReferencesToBundleVariables/bar/artifacts",
   "current_user": {
+    "id": "$USER.Id",
     "short_name": "$USERNAME",
     "userName": "$USERNAME"
   },
diff --git a/acceptance/server_test.go b/acceptance/server_test.go
index 98e351739..4957a7668 100644
--- a/acceptance/server_test.go
+++ b/acceptance/server_test.go
@@ -54,6 +54,7 @@ func AddHandlers(server *testserver.Server) {
 
 	server.Handle("GET /api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, error) {
 		return iam.User{
+			Id:       "1000012345",
 			UserName: "tester@databricks.com",
 		}, nil
 	})

From a03ea730112c7f51c8fc82d55ab8d11908ef8e9f Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 30 Jan 2025 10:52:41 +0100
Subject: [PATCH 177/247] Add ruff.toml with increased line-length (#2268)

The default is 88 which reformats too much.

This has no effect on templates but affects Python script in this PR
https://github.com/databricks/cli/pull/2267

For context, we do not set any line length for golang and have 177 .go
files with max line length 150 or more.
---
 ruff.toml | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 ruff.toml

diff --git a/ruff.toml b/ruff.toml
new file mode 100644
index 000000000..802a3ca67
--- /dev/null
+++ b/ruff.toml
@@ -0,0 +1 @@
+line-length = 150

From f1efbd7d9fb19c4f9457999e5ab8740293f34502 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 30 Jan 2025 11:38:54 +0100
Subject: [PATCH 178/247] acc: add -norepl flag that disables replacements (for
 debugging) (#2269)

---
 acceptance/acceptance_test.go | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 91ad09e9e..e7104a1c1 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -25,7 +25,10 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-var KeepTmp bool
+var (
+	KeepTmp bool
+	NoRepl  bool
+)
 
 // In order to debug CLI running under acceptance test, set this to full subtest name, e.g. "bundle/variables/empty"
 // Then install your breakpoints and click "debug test" near TestAccept in VSCODE.
@@ -40,6 +43,7 @@ var InprocessMode bool
 func init() {
 	flag.BoolVar(&InprocessMode, "inprocess", SingleTest != "", "Run CLI in the same process as test (for debugging)")
 	flag.BoolVar(&KeepTmp, "keeptmp", false, "Do not delete TMP directory after run")
+	flag.BoolVar(&NoRepl, "norepl", false, "Do not apply any replacements (for debugging)")
 }
 
 const (
@@ -272,7 +276,9 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN
 
 	// Apply replacements to the new value only.
 	// The reference value is stored after applying replacements.
-	valueNew = repls.Replace(valueNew)
+	if !NoRepl {
+		valueNew = repls.Replace(valueNew)
+	}
 
 	// The test did not produce an expected output file.
 	if okRef && !okNew {

From 3c6eacb05b6980c59324b1da4ac091f22cb5733d Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Thu, 30 Jan 2025 16:13:07 +0530
Subject: [PATCH 179/247] Add feature to mock server APIs in acceptance tests
 (#2226)

## Changes
This PR allows us to define custom server stubs in a `test.toml` file.

Note: A followup PR will add functionality to do assertions on the API
request itself.

## Tests
New acceptance test.
---
 acceptance/acceptance_test.go               | 25 +++++++++++++++++----
 acceptance/config_test.go                   | 23 +++++++++++++++++++
 acceptance/workspace/jobs/create/output.txt |  5 +++++
 acceptance/workspace/jobs/create/script     |  1 +
 acceptance/workspace/jobs/create/test.toml  |  7 ++++++
 5 files changed, 57 insertions(+), 4 deletions(-)
 create mode 100644 acceptance/workspace/jobs/create/output.txt
 create mode 100644 acceptance/workspace/jobs/create/script
 create mode 100644 acceptance/workspace/jobs/create/test.toml

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index e7104a1c1..60f7945df 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -6,6 +6,7 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -112,10 +113,10 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	cloudEnv := os.Getenv("CLOUD_ENV")
 
 	if cloudEnv == "" {
-		server := testserver.New(t)
-		AddHandlers(server)
+		defaultServer := testserver.New(t)
+		AddHandlers(defaultServer)
 		// Redirect API access to local server:
-		t.Setenv("DATABRICKS_HOST", server.URL)
+		t.Setenv("DATABRICKS_HOST", defaultServer.URL)
 		t.Setenv("DATABRICKS_TOKEN", "dapi1234")
 
 		homeDir := t.TempDir()
@@ -214,6 +215,22 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 
 	args := []string{"bash", "-euo", "pipefail", EntryPointScript}
 	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Env = os.Environ()
+
+	// Start a new server with a custom configuration if the acceptance test
+	// specifies a custom server stubs.
+	if len(config.Server) > 0 {
+		server := testserver.New(t)
+
+		for _, stub := range config.Server {
+			require.NotEmpty(t, stub.Pattern)
+			server.Handle(stub.Pattern, func(req *http.Request) (resp any, err error) {
+				return stub.Response.Body, nil
+			})
+		}
+		cmd.Env = append(cmd.Env, "DATABRICKS_HOST="+server.URL)
+	}
+
 	if coverDir != "" {
 		// Creating individual coverage directory for each test, because writing to the same one
 		// results in sporadic failures like this one (only if tests are running in parallel):
@@ -221,7 +238,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		coverDir = filepath.Join(coverDir, strings.ReplaceAll(dir, string(os.PathSeparator), "--"))
 		err := os.MkdirAll(coverDir, os.ModePerm)
 		require.NoError(t, err)
-		cmd.Env = append(os.Environ(), "GOCOVERDIR="+coverDir)
+		cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir)
 	}
 
 	// Write combined output to a file
diff --git a/acceptance/config_test.go b/acceptance/config_test.go
index 41866c4a7..f340f0367 100644
--- a/acceptance/config_test.go
+++ b/acceptance/config_test.go
@@ -29,6 +29,29 @@ type TestConfig struct {
 	// List of additional replacements to apply on this test.
 	// Old is a regexp, New is a replacement expression.
 	Repls []testdiff.Replacement
+
+	// List of server stubs to load. Example configuration:
+	//
+	// [[Server]]
+	// Pattern = "POST /api/2.1/jobs/create"
+	// Response.Body = '''
+	// {
+	// 	"job_id": 1111
+	// }
+	// '''
+	Server []ServerStub
+}
+
+type ServerStub struct {
+	// The HTTP method and path to match. Examples:
+	// 1. /api/2.0/clusters/list (matches all methods)
+	// 2. GET /api/2.0/clusters/list
+	Pattern string
+
+	// The response body to return.
+	Response struct {
+		Body string
+	}
 }
 
 // FindConfig finds the closest config file.
diff --git a/acceptance/workspace/jobs/create/output.txt b/acceptance/workspace/jobs/create/output.txt
new file mode 100644
index 000000000..a9487fe5b
--- /dev/null
+++ b/acceptance/workspace/jobs/create/output.txt
@@ -0,0 +1,5 @@
+
+>>> $CLI jobs create --json {"name":"abc"}
+{
+  "job_id":1111
+}
diff --git a/acceptance/workspace/jobs/create/script b/acceptance/workspace/jobs/create/script
new file mode 100644
index 000000000..9ff7b5b87
--- /dev/null
+++ b/acceptance/workspace/jobs/create/script
@@ -0,0 +1 @@
+trace $CLI jobs create --json '{"name":"abc"}'
diff --git a/acceptance/workspace/jobs/create/test.toml b/acceptance/workspace/jobs/create/test.toml
new file mode 100644
index 000000000..94e5eee13
--- /dev/null
+++ b/acceptance/workspace/jobs/create/test.toml
@@ -0,0 +1,7 @@
+[[Server]]
+Pattern = "POST /api/2.1/jobs/create"
+Response.Body = '''
+{
+    "job_id": 1111
+}
+'''

From 787dbe909912ab5c4894cb21218cb39d6626e95a Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Fri, 31 Jan 2025 19:01:23 +0530
Subject: [PATCH 180/247] Add request body assertions to acceptance tests
 (#2263)

## Changes
With this PR, any acceptance tests that define custom server stubs in
`test.toml` will automatically record all HTTP requests made and assert
on them.

Builds on top of https://github.com/databricks/cli/pull/2226

## Tests
Modifying existing acceptance test.
---
 acceptance/acceptance_test.go                 | 37 ++++++++++++++++++-
 acceptance/config_test.go                     |  4 ++
 .../workspace/jobs/create/out.requests.txt    |  1 +
 acceptance/workspace/jobs/create/test.toml    |  2 +
 libs/testserver/server.go                     | 25 +++++++++++++
 5 files changed, 67 insertions(+), 2 deletions(-)
 create mode 100644 acceptance/workspace/jobs/create/out.requests.txt

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 60f7945df..bb1d0f44f 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -2,6 +2,7 @@ package acceptance_test
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -219,8 +220,21 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 
 	// Start a new server with a custom configuration if the acceptance test
 	// specifies a custom server stubs.
-	if len(config.Server) > 0 {
-		server := testserver.New(t)
+	var server *testserver.Server
+
+	// Start a new server for this test if either:
+	// 1. A custom server spec is defined in the test configuration.
+	// 2. The test is configured to record requests and assert on them. We need
+	//    a duplicate of the default server to record requests because the default
+	//    server otherwise is a shared resource.
+	if len(config.Server) > 0 || config.RecordRequests {
+		server = testserver.New(t)
+		server.RecordRequests = config.RecordRequests
+
+		// If no custom server stubs are defined, add the default handlers.
+		if len(config.Server) == 0 {
+			AddHandlers(server)
+		}
 
 		for _, stub := range config.Server {
 			require.NotEmpty(t, stub.Pattern)
@@ -249,6 +263,25 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 	cmd.Dir = tmpDir
 	err = cmd.Run()
 
+	// Write the requests made to the server to a output file if the test is
+	// configured to record requests.
+	if config.RecordRequests {
+		f, err := os.OpenFile(filepath.Join(tmpDir, "out.requests.txt"), os.O_CREATE|os.O_WRONLY, 0o644)
+		require.NoError(t, err)
+
+		for _, req := range server.Requests {
+			reqJson, err := json.Marshal(req)
+			require.NoError(t, err)
+
+			line := fmt.Sprintf("%s\n", reqJson)
+			_, err = f.WriteString(line)
+			require.NoError(t, err)
+		}
+
+		err = f.Close()
+		require.NoError(t, err)
+	}
+
 	// Include exit code in output (if non-zero)
 	formatOutput(out, err)
 	require.NoError(t, out.Close())
diff --git a/acceptance/config_test.go b/acceptance/config_test.go
index f340f0367..beceb6a08 100644
--- a/acceptance/config_test.go
+++ b/acceptance/config_test.go
@@ -40,6 +40,10 @@ type TestConfig struct {
 	// }
 	// '''
 	Server []ServerStub
+
+	// Record the requests made to the server and write them as output to
+	// out.requests.txt
+	RecordRequests bool
 }
 
 type ServerStub struct {
diff --git a/acceptance/workspace/jobs/create/out.requests.txt b/acceptance/workspace/jobs/create/out.requests.txt
new file mode 100644
index 000000000..b22876b70
--- /dev/null
+++ b/acceptance/workspace/jobs/create/out.requests.txt
@@ -0,0 +1 @@
+{"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}}
diff --git a/acceptance/workspace/jobs/create/test.toml b/acceptance/workspace/jobs/create/test.toml
index 94e5eee13..1bf36547b 100644
--- a/acceptance/workspace/jobs/create/test.toml
+++ b/acceptance/workspace/jobs/create/test.toml
@@ -1,3 +1,5 @@
+RecordRequests = true
+
 [[Server]]
 Pattern = "POST /api/2.1/jobs/create"
 Response.Body = '''
diff --git a/libs/testserver/server.go b/libs/testserver/server.go
index 9ebfe3ba0..2e8dbdfda 100644
--- a/libs/testserver/server.go
+++ b/libs/testserver/server.go
@@ -2,9 +2,12 @@ package testserver
 
 import (
 	"encoding/json"
+	"io"
 	"net/http"
 	"net/http/httptest"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/databricks/cli/internal/testutil"
 )
 
@@ -13,6 +16,16 @@ type Server struct {
 	Mux *http.ServeMux
 
 	t testutil.TestingT
+
+	RecordRequests bool
+
+	Requests []Request
+}
+
+type Request struct {
+	Method string `json:"method"`
+	Path   string `json:"path"`
+	Body   any    `json:"body"`
 }
 
 func New(t testutil.TestingT) *Server {
@@ -37,6 +50,18 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) {
 			return
 		}
 
+		if s.RecordRequests {
+			body, err := io.ReadAll(r.Body)
+			assert.NoError(s.t, err)
+
+			s.Requests = append(s.Requests, Request{
+				Method: r.Method,
+				Path:   r.URL.Path,
+				Body:   json.RawMessage(body),
+			})
+
+		}
+
 		w.Header().Set("Content-Type", "application/json")
 
 		var respBytes []byte

From e5730bf57ecb39049047ba394734831589a81bbb Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 31 Jan 2025 14:53:13 +0100
Subject: [PATCH 181/247] Use real terraform in acceptance tests (#2267)

## Changes
- Add a script install_terraform.py that downloads terraform and
provider and generates a config to use, inspired by
https://gist.github.com/pietern/1cb6b6f3e0a452328e13cdc75031105e
- Make acceptance tests run this script once before running the tests
and set the required env vars to make cli use this terraform
installation.
- Use OS-specific directory for things that are build by acceptance test
runner (CLI and terraform).

This enables acceptance tests against cloud #2242 and local test for
bundle deploy #2254.

## Tests
- Add an acceptance test for standalone terraform. This is useful to
debug terraform with TF_LOG=DEBUG to see that it uses local provider.
- Other acceptance tests are updated with regard to terraform exec path.
- The overall time for tests locally is unchanged (if terraform is
already fetched).
---
 acceptance/.gitignore                         |   1 +
 acceptance/acceptance_test.go                 |  61 ++++++---
 acceptance/build/.gitignore                   |   1 -
 .../bundle/variables/git-branch/output.txt    |   4 +-
 .../prepend-workspace-var/output.txt          |   2 +-
 acceptance/install_terraform.py               | 122 ++++++++++++++++++
 acceptance/terraform/main.tf                  |  25 ++++
 acceptance/terraform/output.txt               |  51 ++++++++
 acceptance/terraform/script                   |  14 ++
 9 files changed, 256 insertions(+), 25 deletions(-)
 create mode 100644 acceptance/.gitignore
 delete mode 100644 acceptance/build/.gitignore
 create mode 100755 acceptance/install_terraform.py
 create mode 100644 acceptance/terraform/main.tf
 create mode 100644 acceptance/terraform/output.txt
 create mode 100644 acceptance/terraform/script

diff --git a/acceptance/.gitignore b/acceptance/.gitignore
new file mode 100644
index 000000000..378eac25d
--- /dev/null
+++ b/acceptance/.gitignore
@@ -0,0 +1 @@
+build
diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index bb1d0f44f..5f0030eec 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -77,6 +77,11 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	cwd, err := os.Getwd()
 	require.NoError(t, err)
 
+	buildDir := filepath.Join(cwd, "build", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH))
+
+	// Download terraform and provider and create config; this also creates build directory.
+	RunCommand(t, []string{"python3", filepath.Join(cwd, "install_terraform.py"), "--targetdir", buildDir}, ".")
+
 	coverDir := os.Getenv("CLI_GOCOVERDIR")
 
 	if coverDir != "" {
@@ -93,7 +98,7 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 		t.Setenv("CMD_SERVER_URL", cmdServer.URL)
 		execPath = filepath.Join(cwd, "bin", "callserver.py")
 	} else {
-		execPath = BuildCLI(t, cwd, coverDir)
+		execPath = BuildCLI(t, buildDir, coverDir)
 	}
 
 	t.Setenv("CLI", execPath)
@@ -123,11 +128,24 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 		homeDir := t.TempDir()
 		// Do not read user's ~/.databrickscfg
 		t.Setenv(env.HomeEnvVar(), homeDir)
-
-		// Prevent CLI from downloading terraform in each test:
-		t.Setenv("DATABRICKS_TF_EXEC_PATH", tempHomeDir)
 	}
 
+	terraformrcPath := filepath.Join(buildDir, ".terraformrc")
+	t.Setenv("TF_CLI_CONFIG_FILE", terraformrcPath)
+	t.Setenv("DATABRICKS_TF_CLI_CONFIG_FILE", terraformrcPath)
+	repls.SetPath(terraformrcPath, "$DATABRICKS_TF_CLI_CONFIG_FILE")
+
+	terraformExecPath := filepath.Join(buildDir, "terraform")
+	if runtime.GOOS == "windows" {
+		terraformExecPath += ".exe"
+	}
+	t.Setenv("DATABRICKS_TF_EXEC_PATH", terraformExecPath)
+	t.Setenv("TERRAFORM", terraformExecPath)
+	repls.SetPath(terraformExecPath, "$TERRAFORM")
+
+	// do it last so that full paths match first:
+	repls.SetPath(buildDir, "$BUILD_DIR")
+
 	workspaceClient, err := databricks.NewWorkspaceClient()
 	require.NoError(t, err)
 
@@ -406,13 +424,12 @@ func readMergedScriptContents(t *testing.T, dir string) string {
 	return strings.Join(prepares, "\n")
 }
 
-func BuildCLI(t *testing.T, cwd, coverDir string) string {
-	execPath := filepath.Join(cwd, "build", "databricks")
+func BuildCLI(t *testing.T, buildDir, coverDir string) string {
+	execPath := filepath.Join(buildDir, "databricks")
 	if runtime.GOOS == "windows" {
 		execPath += ".exe"
 	}
 
-	start := time.Now()
 	args := []string{
 		"go", "build",
 		"-mod", "vendor",
@@ -430,20 +447,8 @@ func BuildCLI(t *testing.T, cwd, coverDir string) string {
 		args = append(args, "-buildvcs=false")
 	}
 
-	cmd := exec.Command(args[0], args[1:]...)
-	cmd.Dir = ".."
-	out, err := cmd.CombinedOutput()
-	elapsed := time.Since(start)
-	t.Logf("%s took %s", args, elapsed)
-	require.NoError(t, err, "go build failed: %s: %s\n%s", args, err, out)
-	if len(out) > 0 {
-		t.Logf("go build output: %s: %s", args, out)
-	}
-
-	// Quick check + warm up cache:
-	cmd = exec.Command(execPath, "--version")
-	out, err = cmd.CombinedOutput()
-	require.NoError(t, err, "%s --version failed: %s\n%s", execPath, err, out)
+	RunCommand(t, args, "..")
+	RunCommand(t, []string{execPath, "--version"}, ".")
 	return execPath
 }
 
@@ -581,3 +586,17 @@ func getUVDefaultCacheDir(t *testing.T) string {
 		return cacheDir + "/uv"
 	}
 }
+
+func RunCommand(t *testing.T, args []string, dir string) {
+	start := time.Now()
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Dir = dir
+	out, err := cmd.CombinedOutput()
+	elapsed := time.Since(start)
+	t.Logf("%s took %s", args, elapsed)
+
+	require.NoError(t, err, "%s failed: %s\n%s", args, err, out)
+	if len(out) > 0 {
+		t.Logf("%s output: %s", args, out)
+	}
+}
diff --git a/acceptance/build/.gitignore b/acceptance/build/.gitignore
deleted file mode 100644
index a48b4db25..000000000
--- a/acceptance/build/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-databricks
diff --git a/acceptance/bundle/variables/git-branch/output.txt b/acceptance/bundle/variables/git-branch/output.txt
index fb3ab805a..5e7664f61 100644
--- a/acceptance/bundle/variables/git-branch/output.txt
+++ b/acceptance/bundle/variables/git-branch/output.txt
@@ -11,7 +11,7 @@
     "name": "git",
     "target": "prod",
     "terraform": {
-      "exec_path": "$TMPHOME"
+      "exec_path": "$TERRAFORM"
     }
   },
   "sync": {
@@ -61,7 +61,7 @@ Validation OK!
     "name": "git",
     "target": "dev",
     "terraform": {
-      "exec_path": "$TMPHOME"
+      "exec_path": "$TERRAFORM"
     }
   },
   "sync": {
diff --git a/acceptance/bundle/variables/prepend-workspace-var/output.txt b/acceptance/bundle/variables/prepend-workspace-var/output.txt
index fcaa25b4a..ed6c2b2af 100644
--- a/acceptance/bundle/variables/prepend-workspace-var/output.txt
+++ b/acceptance/bundle/variables/prepend-workspace-var/output.txt
@@ -7,7 +7,7 @@
     },
     "target": "dev",
     "terraform": {
-      "exec_path": "$TMPHOME"
+      "exec_path": "$TERRAFORM"
     }
   },
   "resources": {
diff --git a/acceptance/install_terraform.py b/acceptance/install_terraform.py
new file mode 100755
index 000000000..4cf6a9729
--- /dev/null
+++ b/acceptance/install_terraform.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""
+Script to set up terraform and databricks terraform provider in a local directory:
+
+- Download terraform.
+- Download databricks provider.
+- Write a .terraformrc config file that uses this directory.
+- The config file contains env vars that need to be set so that databricks CLI uses this terraform and provider.
+"""
+
+import os
+import platform
+import zipfile
+import argparse
+import json
+from pathlib import Path
+from urllib.request import urlretrieve
+
+os_name = platform.system().lower()
+
+arch = platform.machine().lower()
+arch = {"x86_64": "amd64"}.get(arch, arch)
+if os_name == "windows" and arch not in ("386", "amd64"):
+    # terraform 1.5.5 only has builds for these two.
+    arch = "amd64"
+
+terraform_version = "1.5.5"
+terraform_file = f"terraform_{terraform_version}_{os_name}_{arch}.zip"
+terraform_url = f"https://releases.hashicorp.com/terraform/{terraform_version}/{terraform_file}"
+terraform_binary = "terraform.exe" if os_name == "windows" else "terraform"
+
+
+def retrieve(url, path):
+    if not path.exists():
+        print(f"Downloading {url} -> {path}")
+        urlretrieve(url, path)
+
+
+def read_version(path):
+    for line in path.open():
+        if "ProviderVersion" in line:
+            # Expecting 'const ProviderVersion = "1.64.1"'
+            items = line.strip().split()
+            assert len(items) >= 3, items
+            assert items[-3:-1] == ["ProviderVersion", "="], items
+            version = items[-1].strip('"')
+            assert version, items
+            return version
+    raise SystemExit(f"Could not find ProviderVersion in {path}")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--targetdir", default="build", type=Path)
+    parser.add_argument("--provider-version")
+    args = parser.parse_args()
+    target = args.targetdir
+
+    if not args.provider_version:
+        version_file = Path(__file__).parent.parent / "bundle/internal/tf/codegen/schema/version.go"
+        assert version_file.exists(), version_file
+        terraform_provider_version = read_version(version_file)
+        print(f"Read version {terraform_provider_version} from {version_file}")
+    else:
+        terraform_provider_version = args.provider_version
+
+    terraform_provider_file = f"terraform-provider-databricks_{terraform_provider_version}_{os_name}_{arch}.zip"
+    terraform_provider_url = (
+        f"https://github.com/databricks/terraform-provider-databricks/releases/download/v{terraform_provider_version}/{terraform_provider_file}"
+    )
+
+    target.mkdir(exist_ok=True, parents=True)
+
+    zip_path = target / terraform_file
+    terraform_path = target / terraform_binary
+    terraform_provider_path = target / terraform_provider_file
+
+    retrieve(terraform_url, zip_path)
+    retrieve(terraform_provider_url, terraform_provider_path)
+
+    if not terraform_path.exists():
+        print(f"Extracting {zip_path} -> {terraform_path}")
+
+        with zipfile.ZipFile(zip_path, "r") as zip_ref:
+            zip_ref.extractall(target)
+
+        terraform_path.chmod(0o755)
+
+    tfplugins_path = target / "tfplugins"
+    provider_dir = Path(tfplugins_path / f"registry.terraform.io/databricks/databricks/{terraform_provider_version}/{os_name}_{arch}")
+    if not provider_dir.exists():
+        print(f"Extracting {terraform_provider_path} -> {provider_dir}")
+        os.makedirs(provider_dir, exist_ok=True)
+        with zipfile.ZipFile(terraform_provider_path, "r") as zip_ref:
+            zip_ref.extractall(provider_dir)
+
+        files = list(provider_dir.iterdir())
+        assert files, provider_dir
+
+        for f in files:
+            f.chmod(0o755)
+
+    terraformrc_path = target / ".terraformrc"
+    if not terraformrc_path.exists():
+        path = json.dumps(str(tfplugins_path.absolute()))
+        text = f"""# Set these env variables before running databricks cli:
+# export DATABRICKS_TF_CLI_CONFIG_FILE={terraformrc_path.absolute()}
+# export DATABRICKS_TF_EXEC_PATH={terraform_path.absolute()}
+
+provider_installation {{
+    filesystem_mirror {{
+        path = {path}
+        include = ["registry.terraform.io/databricks/databricks"]
+    }}
+}}
+"""
+        print(f"Writing {terraformrc_path}:\n{text}")
+        terraformrc_path.write_text(text)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/acceptance/terraform/main.tf b/acceptance/terraform/main.tf
new file mode 100644
index 000000000..93f665ff4
--- /dev/null
+++ b/acceptance/terraform/main.tf
@@ -0,0 +1,25 @@
+terraform {
+  required_providers {
+    databricks = {
+      source  = "databricks/databricks"
+      version = "1.64.1"
+    }
+  }
+
+  required_version = "= 1.5.5"
+}
+
+provider "databricks" {
+  # Optionally, specify the Databricks host and token
+  # host  = "https://<your-databricks-instance>"
+  # token = "<YOUR_PERSONAL_ACCESS_TOKEN>"
+}
+
+data "databricks_current_user" "me" {
+  # Retrieves the current user's information
+}
+
+output "username" {
+  description = "Username"
+  value       = "${data.databricks_current_user.me.user_name}"
+}
diff --git a/acceptance/terraform/output.txt b/acceptance/terraform/output.txt
new file mode 100644
index 000000000..c3d453ea5
--- /dev/null
+++ b/acceptance/terraform/output.txt
@@ -0,0 +1,51 @@
+
+>>> $TERRAFORM init -no-color -get=false
+
+Initializing the backend...
+
+Initializing provider plugins...
+- Finding databricks/databricks versions matching "1.64.1"...
+- Installing databricks/databricks v1.64.1...
+- Installed databricks/databricks v1.64.1 (unauthenticated)
+
+Terraform has created a lock file .terraform.lock.hcl to record the provider
+selections it made above. Include this file in your version control repository
+so that Terraform can guarantee to make the same selections by default when
+you run "terraform init" in the future.
+
+
+Warning: Incomplete lock file information for providers
+
+Due to your customized provider installation methods, Terraform was forced to
+calculate lock file checksums locally for the following providers:
+  - databricks/databricks
+
+
+To calculate additional checksums for another platform, run:
+  terraform providers lock -platform=linux_amd64
+(where linux_amd64 is the platform to generate)
+
+Terraform has been successfully initialized!
+
+You may now begin working with Terraform. Try running "terraform plan" to see
+any changes that are required for your infrastructure. All Terraform commands
+should now work.
+
+If you ever set or change modules or backend configuration for Terraform,
+rerun this command to reinitialize your working directory. If you forget, other
+commands will detect it and remind you to do so if necessary.
+
+>>> $TERRAFORM plan -no-color
+data.databricks_current_user.me: Reading...
+data.databricks_current_user.me: Read complete after 0s [id=$USER.Id]
+
+Changes to Outputs:
+  + username = "$USERNAME"
+
+You can apply this plan to save these new output values to the Terraform
+state, without changing any real infrastructure.
+
+─────────────────────────────────────────────────────────────────────────────
+
+Note: You didn't use the -out option to save this plan, so Terraform can't
+guarantee to take exactly these actions if you run "terraform apply" now.
diff --git a/acceptance/terraform/script b/acceptance/terraform/script
new file mode 100644
index 000000000..78e35049d
--- /dev/null
+++ b/acceptance/terraform/script
@@ -0,0 +1,14 @@
+# Want to filter out these message:
+# Mac:
+# The current .terraform.lock.hcl file only includes checksums for
+# darwin_arm64, so Terraform running on another platform will fail to install
+# these providers.
+#
+# Linux:
+# The current .terraform.lock.hcl file only includes checksums for linux_amd64,
+# so Terraform running on another platform will fail to install these
+# providers.
+
+trace $TERRAFORM init -no-color -get=false | grep -v 'includes checksums for' | grep -v 'so Terraform running on another' | grep -v 'providers\.'
+trace $TERRAFORM plan -no-color
+rm -fr .terraform.lock.hcl .terraform

From 2f798c4dedd7e4fc18d9f31cbd55135345e1b208 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 3 Feb 2025 11:03:18 +0100
Subject: [PATCH 182/247] acc: Remove initial '$CLI --version' call (#2280)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

It is proven to be not necessary.

```
~/work/cli/acceptance % hyperfine -w 2 'go test'  # with change:
Benchmark 1: go test
  Time (mean ± σ):      4.983 s ±  0.209 s    [User: 6.073 s, System: 9.869 s]
  Range (min … max):    4.792 s …  5.483 s    10 runs

~/work/cli/acceptance % git stash  # without change:
~/work/cli/acceptance % hyperfine -w 2 'go test'
Benchmark 1: go test
  Time (mean ± σ):      5.018 s ±  0.100 s    [User: 6.142 s, System: 10.234 s]
  Range (min … max):    4.899 s …  5.182 s    10 runs
```
---
 acceptance/acceptance_test.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 5f0030eec..571168ca8 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -448,7 +448,6 @@ func BuildCLI(t *testing.T, buildDir, coverDir string) string {
 	}
 
 	RunCommand(t, args, "..")
-	RunCommand(t, []string{execPath, "--version"}, ".")
 	return execPath
 }
 

From fcedfe4c781a1f6de8f23d6e477ed045d91921ea Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 3 Feb 2025 11:29:13 +0100
Subject: [PATCH 183/247] acc: Consistent & detailed output for file issues
 (#2279)

## Changes
- Include compact relPath in the error message title. Include full paths
in separate lines below.
- Previously sometimes full paths were printed, sometime only rel path.

## Tests
Manually trigger the errors.
---
 acceptance/acceptance_test.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 571168ca8..cad8acf4f 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -335,7 +335,7 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN
 	bufRef, okRef := tryReading(t, pathRef)
 	bufNew, okNew := tryReading(t, pathNew)
 	if !okRef && !okNew {
-		t.Errorf("Both files are missing or have errors: %s, %s", pathRef, pathNew)
+		t.Errorf("Both files are missing or have errors: %s\npathRef: %s\npathNew: %s", relPath, pathRef, pathNew)
 		return
 	}
 
@@ -350,7 +350,7 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN
 
 	// The test did not produce an expected output file.
 	if okRef && !okNew {
-		t.Errorf("Missing output file: %s", relPath)
+		t.Errorf("Missing output file: %s\npathRef: %s\npathNew: %s", relPath, pathRef, pathNew)
 		testdiff.AssertEqualTexts(t, pathRef, pathNew, valueRef, valueNew)
 		if testdiff.OverwriteMode {
 			t.Logf("Removing output file: %s", relPath)
@@ -361,7 +361,7 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN
 
 	// The test produced an unexpected output file.
 	if !okRef && okNew {
-		t.Errorf("Unexpected output file: %s", relPath)
+		t.Errorf("Unexpected output file: %s\npathRef: %s\npathNew: %s", relPath, pathRef, pathNew)
 		testdiff.AssertEqualTexts(t, pathRef, pathNew, valueRef, valueNew)
 		if testdiff.OverwriteMode {
 			t.Logf("Writing output file: %s", relPath)

From f267318bb940849c8833308128ff9eaf08c35aa6 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 3 Feb 2025 11:43:25 +0100
Subject: [PATCH 184/247] Include acceptance tests in integration tests (#2242)

## Changes
- Include acceptance directory in integration tests. Acceptance tests
will not start local server if CLOUD_ENV is set, so they become
integration tests.
- Add dependency for vendor to integration, so that CLI can be build
there.
- Implement LocalOnly option in test.toml to opt out of running
acceptance tests as integration tests. Use it in certain tests that are
difficult or not necessary to fix when run as integration tests.
- Update terraform test to redact timings out.
- Clean up .workspace.current_user from outputs of the tests.

## Tests
Existing tests.
---
 Makefile                                           |  6 +++---
 acceptance/acceptance_test.go                      |  5 +++++
 acceptance/bundle/scripts/test.toml                |  1 +
 acceptance/bundle/templates/test.toml              |  2 ++
 .../bundle/variables/env_overrides/test.toml       |  2 ++
 acceptance/bundle/variables/git-branch/output.txt  | 14 ++------------
 acceptance/bundle/variables/git-branch/script      |  6 +++---
 .../variables/prepend-workspace-var/output.txt     |  7 +------
 .../bundle/variables/prepend-workspace-var/script  |  2 +-
 .../bundle/variables/resolve-builtin/output.txt    |  5 -----
 acceptance/bundle/variables/resolve-builtin/script |  2 +-
 .../bundle/variables/resolve-builtin/test.toml     |  2 ++
 .../variables/resolve-vars-in-root-path/output.txt |  5 -----
 .../variables/resolve-vars-in-root-path/script     |  2 +-
 .../variables/resolve-vars-in-root-path/test.toml  |  2 ++
 acceptance/config_test.go                          |  3 +++
 acceptance/terraform/output.txt                    |  2 +-
 acceptance/terraform/test.toml                     |  3 +++
 acceptance/workspace/jobs/create/test.toml         |  1 +
 19 files changed, 34 insertions(+), 38 deletions(-)
 create mode 100644 acceptance/bundle/scripts/test.toml
 create mode 100644 acceptance/bundle/templates/test.toml
 create mode 100644 acceptance/bundle/variables/env_overrides/test.toml
 create mode 100644 acceptance/bundle/variables/resolve-builtin/test.toml
 create mode 100644 acceptance/bundle/variables/resolve-vars-in-root-path/test.toml
 create mode 100644 acceptance/terraform/test.toml

diff --git a/Makefile b/Makefile
index d30ccef14..e18727934 100644
--- a/Makefile
+++ b/Makefile
@@ -51,12 +51,12 @@ schema:
 docs:
 	go run ./bundle/docsgen ./bundle/internal/schema ./bundle/docsgen
 
-INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./integration/..." -- -parallel 4 -timeout=2h
+INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./acceptance ./integration/..." -- -parallel 4 -timeout=2h
 
-integration:
+integration: vendor
 	$(INTEGRATION)
 
-integration-short:
+integration-short: vendor
 	$(INTEGRATION) -short
 
 .PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover docs
diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index cad8acf4f..ccdf74bcb 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -209,6 +209,11 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		t.Skipf("Disabled via GOOS.%s setting in %s", runtime.GOOS, configPath)
 	}
 
+	cloudEnv := os.Getenv("CLOUD_ENV")
+	if config.LocalOnly && cloudEnv != "" {
+		t.Skipf("Disabled via LocalOnly setting in %s (CLOUD_ENV=%s)", configPath, cloudEnv)
+	}
+
 	var tmpDir string
 	var err error
 	if KeepTmp {
diff --git a/acceptance/bundle/scripts/test.toml b/acceptance/bundle/scripts/test.toml
new file mode 100644
index 000000000..1dbd78681
--- /dev/null
+++ b/acceptance/bundle/scripts/test.toml
@@ -0,0 +1 @@
+LocalOnly = true  # Deployment currently fails when run locally; once that is fixed, remove this setting
diff --git a/acceptance/bundle/templates/test.toml b/acceptance/bundle/templates/test.toml
new file mode 100644
index 000000000..90539263d
--- /dev/null
+++ b/acceptance/bundle/templates/test.toml
@@ -0,0 +1,2 @@
+# At the moment, there are many differences across different envs w.r.t to catalog use, node type and so on.
+LocalOnly = true
diff --git a/acceptance/bundle/variables/env_overrides/test.toml b/acceptance/bundle/variables/env_overrides/test.toml
new file mode 100644
index 000000000..439c2fab1
--- /dev/null
+++ b/acceptance/bundle/variables/env_overrides/test.toml
@@ -0,0 +1,2 @@
+# Cloud run fails with Error: failed to resolve cluster-policy: wrong-cluster-policy, err: Policy named 'wrong-cluster-policy' does not exist
+LocalOnly = true
diff --git a/acceptance/bundle/variables/git-branch/output.txt b/acceptance/bundle/variables/git-branch/output.txt
index 5e7664f61..21ed9e7de 100644
--- a/acceptance/bundle/variables/git-branch/output.txt
+++ b/acceptance/bundle/variables/git-branch/output.txt
@@ -6,7 +6,7 @@
     "git": {
       "actual_branch": "main",
       "branch": "",
-      "bundle_root_path": ".",
+      "bundle_root_path": "."
     },
     "name": "git",
     "target": "prod",
@@ -28,11 +28,6 @@
   },
   "workspace": {
     "artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/artifacts",
-    "current_user": {
-      "id": "$USER.Id",
-      "short_name": "$USERNAME",
-      "userName": "$USERNAME"
-    },
     "file_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/files",
     "resource_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/resources",
     "root_path": "/Workspace/Users/$USERNAME/.bundle/git/prod",
@@ -56,7 +51,7 @@ Validation OK!
     "git": {
       "actual_branch": "main",
       "branch": "dev-branch",
-      "bundle_root_path": ".",
+      "bundle_root_path": "."
     },
     "name": "git",
     "target": "dev",
@@ -78,11 +73,6 @@ Validation OK!
   },
   "workspace": {
     "artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/artifacts",
-    "current_user": {
-      "id": "$USER.Id",
-      "short_name": "$USERNAME",
-      "userName": "$USERNAME"
-    },
     "file_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/files",
     "resource_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/resources",
     "root_path": "/Workspace/Users/$USERNAME/.bundle/git/dev",
diff --git a/acceptance/bundle/variables/git-branch/script b/acceptance/bundle/variables/git-branch/script
index aed881f1f..8f99cc01b 100644
--- a/acceptance/bundle/variables/git-branch/script
+++ b/acceptance/bundle/variables/git-branch/script
@@ -1,6 +1,6 @@
 git-repo-init
-trace $CLI bundle validate -o json | grep -v '"commit"'
+trace $CLI bundle validate -o json | jq 'del(.workspace.current_user, .bundle.git.commit)'
 trace $CLI bundle validate
-trace $CLI bundle validate -o json -t dev | grep -v '"commit"'
-trace $CLI bundle validate -t dev | grep -v '"commit"'
+trace $CLI bundle validate -o json -t dev | jq 'del(.workspace.current_user, .bundle.git.commit)'
+trace $CLI bundle validate -t dev
 rm -fr .git
diff --git a/acceptance/bundle/variables/prepend-workspace-var/output.txt b/acceptance/bundle/variables/prepend-workspace-var/output.txt
index ed6c2b2af..93b652894 100644
--- a/acceptance/bundle/variables/prepend-workspace-var/output.txt
+++ b/acceptance/bundle/variables/prepend-workspace-var/output.txt
@@ -53,15 +53,10 @@
   },
   "workspace": {
     "artifact_path": "/Users/$USERNAME/path/to/root/artifacts",
-    "current_user": {
-      "id": "$USER.Id",
-      "short_name": "$USERNAME",
-      "userName": "$USERNAME"
-    },
     "file_path": "/Users/$USERNAME/path/to/root/files",
     "profile": "profile_name",
     "resource_path": "/Users/$USERNAME/path/to/root/resources",
     "root_path": "/Users/$USERNAME/path/to/root",
     "state_path": "/Users/$USERNAME/path/to/root/state"
   }
-}
\ No newline at end of file
+}
diff --git a/acceptance/bundle/variables/prepend-workspace-var/script b/acceptance/bundle/variables/prepend-workspace-var/script
index de6bc8a17..e30ffb9c4 100644
--- a/acceptance/bundle/variables/prepend-workspace-var/script
+++ b/acceptance/bundle/variables/prepend-workspace-var/script
@@ -1,2 +1,2 @@
 echo /Workspace should be prepended on all paths, but it is not the case: #2181
-$CLI bundle validate -o json
+$CLI bundle validate -o json | jq 'del(.workspace.current_user)'
diff --git a/acceptance/bundle/variables/resolve-builtin/output.txt b/acceptance/bundle/variables/resolve-builtin/output.txt
index 0c1678f84..f37a2a19e 100644
--- a/acceptance/bundle/variables/resolve-builtin/output.txt
+++ b/acceptance/bundle/variables/resolve-builtin/output.txt
@@ -1,10 +1,5 @@
 {
   "artifact_path": "TestResolveVariableReferences/bar/artifacts",
-  "current_user": {
-    "id": "$USER.Id",
-    "short_name": "$USERNAME",
-    "userName": "$USERNAME"
-  },
   "file_path": "TestResolveVariableReferences/bar/baz",
   "resource_path": "TestResolveVariableReferences/bar/resources",
   "root_path": "TestResolveVariableReferences/bar",
diff --git a/acceptance/bundle/variables/resolve-builtin/script b/acceptance/bundle/variables/resolve-builtin/script
index fefd9abe6..558d0a7ca 100644
--- a/acceptance/bundle/variables/resolve-builtin/script
+++ b/acceptance/bundle/variables/resolve-builtin/script
@@ -1 +1 @@
-$CLI bundle validate -o json | jq .workspace
+$CLI bundle validate -o json | jq .workspace | jq 'del(.current_user)'
diff --git a/acceptance/bundle/variables/resolve-builtin/test.toml b/acceptance/bundle/variables/resolve-builtin/test.toml
new file mode 100644
index 000000000..085fab6c0
--- /dev/null
+++ b/acceptance/bundle/variables/resolve-builtin/test.toml
@@ -0,0 +1,2 @@
+# Cloud run fails with Error: Path (TestResolveVariableReferences/bar/baz) doesn't start with '/'
+LocalOnly = true
diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt b/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt
index 51eb40c91..fb828d826 100644
--- a/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt
+++ b/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt
@@ -1,10 +1,5 @@
 {
   "artifact_path": "TestResolveVariableReferencesToBundleVariables/bar/artifacts",
-  "current_user": {
-    "id": "$USER.Id",
-    "short_name": "$USERNAME",
-    "userName": "$USERNAME"
-  },
   "file_path": "TestResolveVariableReferencesToBundleVariables/bar/files",
   "resource_path": "TestResolveVariableReferencesToBundleVariables/bar/resources",
   "root_path": "TestResolveVariableReferencesToBundleVariables/bar",
diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/script b/acceptance/bundle/variables/resolve-vars-in-root-path/script
index fefd9abe6..558d0a7ca 100644
--- a/acceptance/bundle/variables/resolve-vars-in-root-path/script
+++ b/acceptance/bundle/variables/resolve-vars-in-root-path/script
@@ -1 +1 @@
-$CLI bundle validate -o json | jq .workspace
+$CLI bundle validate -o json | jq .workspace | jq 'del(.current_user)'
diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/test.toml b/acceptance/bundle/variables/resolve-vars-in-root-path/test.toml
new file mode 100644
index 000000000..d833bd848
--- /dev/null
+++ b/acceptance/bundle/variables/resolve-vars-in-root-path/test.toml
@@ -0,0 +1,2 @@
+# Cloud run fails with Error: Path (TestResolveVariableReferencesToBundleVariables/bar/files) doesn't start with '/'
+LocalOnly = true
diff --git a/acceptance/config_test.go b/acceptance/config_test.go
index beceb6a08..c7be223de 100644
--- a/acceptance/config_test.go
+++ b/acceptance/config_test.go
@@ -26,6 +26,9 @@ type TestConfig struct {
 	// If absent, default to true.
 	GOOS map[string]bool
 
+	// If true, do not run this test against cloud environment
+	LocalOnly bool
+
 	// List of additional replacements to apply on this test.
 	// Old is a regexp, New is a replacement expression.
 	Repls []testdiff.Replacement
diff --git a/acceptance/terraform/output.txt b/acceptance/terraform/output.txt
index c3d453ea5..32589ddab 100644
--- a/acceptance/terraform/output.txt
+++ b/acceptance/terraform/output.txt
@@ -37,7 +37,7 @@ commands will detect it and remind you to do so if necessary.
 
 >>> $TERRAFORM plan -no-color
 data.databricks_current_user.me: Reading...
-data.databricks_current_user.me: Read complete after 0s [id=$USER.Id]
+data.databricks_current_user.me: Read complete after (redacted) [id=$USER.Id]
 
 Changes to Outputs:
   + username = "$USERNAME"
diff --git a/acceptance/terraform/test.toml b/acceptance/terraform/test.toml
new file mode 100644
index 000000000..a6849e30f
--- /dev/null
+++ b/acceptance/terraform/test.toml
@@ -0,0 +1,3 @@
+[[Repls]]
+Old = 'Read complete after [^\s]+'
+New = 'Read complete after (redacted)'
diff --git a/acceptance/workspace/jobs/create/test.toml b/acceptance/workspace/jobs/create/test.toml
index 1bf36547b..e69569c18 100644
--- a/acceptance/workspace/jobs/create/test.toml
+++ b/acceptance/workspace/jobs/create/test.toml
@@ -1,3 +1,4 @@
+LocalOnly = true  # request recording currently does not work with cloud environment
 RecordRequests = true
 
 [[Server]]

From 91e04cc4441d011a568eaf6b064b85ff6c190d6b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 3 Feb 2025 12:36:30 +0100
Subject: [PATCH 185/247] Bump golangci/golangci-lint-action from 6.1.1 to
 6.2.0 (#2273)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bumps
[golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action)
from 6.1.1 to 6.2.0.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/golangci/golangci-lint-action/releases">golangci/golangci-lint-action's
releases</a>.</em></p>
<blockquote>
<h2>v6.2.0</h2>
<!-- raw HTML omitted -->
<h2>What's Changed</h2>
<h3>Changes</h3>
<ul>
<li>chore: use new build tag syntax by <a
href="https://github.com/alexandear"><code>@​alexandear</code></a> in <a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1133">golangci/golangci-lint-action#1133</a></li>
<li>feat: support linux arm64 public preview by <a
href="https://github.com/ldez"><code>@​ldez</code></a> in <a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1144">golangci/golangci-lint-action#1144</a></li>
</ul>
<h3>Documentation</h3>
<ul>
<li>docs: update local development instructions by <a
href="https://github.com/dmitris"><code>@​dmitris</code></a> in <a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1125">golangci/golangci-lint-action#1125</a></li>
</ul>
<h3>Dependencies</h3>
<ul>
<li>build(deps-dev): bump the dev-dependencies group with 3 updates by
<a href="https://github.com/dependabot"><code>@​dependabot</code></a> in
<a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1112">golangci/golangci-lint-action#1112</a></li>
<li>build(deps): bump the dependencies group with 2 updates by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1113">golangci/golangci-lint-action#1113</a></li>
<li>build(deps-dev): bump the dev-dependencies group with 3 updates by
<a href="https://github.com/dependabot"><code>@​dependabot</code></a> in
<a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1114">golangci/golangci-lint-action#1114</a></li>
<li>build(deps): bump <code>@​types/node</code> from 22.7.4 to 22.7.5 in
the dependencies group by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1115">golangci/golangci-lint-action#1115</a></li>
<li>build(deps-dev): bump the dev-dependencies group with 2 updates by
<a href="https://github.com/dependabot"><code>@​dependabot</code></a> in
<a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1117">golangci/golangci-lint-action#1117</a></li>
<li>build(deps): bump <code>@​types/node</code> from 22.7.5 to 22.7.7 in
the dependencies group by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1118">golangci/golangci-lint-action#1118</a></li>
<li>build(deps-dev): bump the dev-dependencies group with 2 updates by
<a href="https://github.com/dependabot"><code>@​dependabot</code></a> in
<a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1119">golangci/golangci-lint-action#1119</a></li>
<li>build(deps): bump <code>@​types/node</code> from 22.7.7 to 22.8.1 in
the dependencies group by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1120">golangci/golangci-lint-action#1120</a></li>
<li>build(deps-dev): bump the dev-dependencies group with 2 updates by
<a href="https://github.com/dependabot"><code>@​dependabot</code></a> in
<a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1122">golangci/golangci-lint-action#1122</a></li>
<li>build(deps): bump the dependencies group with 2 updates by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1123">golangci/golangci-lint-action#1123</a></li>
<li>build(deps-dev): bump the dev-dependencies group with 2 updates by
<a href="https://github.com/dependabot"><code>@​dependabot</code></a> in
<a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1126">golangci/golangci-lint-action#1126</a></li>
<li>build(deps): bump <code>@​types/node</code> from 22.8.7 to 22.9.0 in
the dependencies group by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1127">golangci/golangci-lint-action#1127</a></li>
<li>build(deps-dev): bump the dev-dependencies group with 3 updates by
<a href="https://github.com/dependabot"><code>@​dependabot</code></a> in
<a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1128">golangci/golangci-lint-action#1128</a></li>
<li>build(deps): bump <code>@​types/node</code> from 22.9.0 to 22.9.3 in
the dependencies group by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1130">golangci/golangci-lint-action#1130</a></li>
<li>build(deps): bump <code>@​types/node</code> from 22.9.3 to 22.10.1
in the dependencies group by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1131">golangci/golangci-lint-action#1131</a></li>
<li>build(deps-dev): bump the dev-dependencies group across 1 directory
with 4 updates by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1132">golangci/golangci-lint-action#1132</a></li>
<li>build(deps-dev): bump the dev-dependencies group with 3 updates by
<a href="https://github.com/dependabot"><code>@​dependabot</code></a> in
<a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1134">golangci/golangci-lint-action#1134</a></li>
<li>build(deps): bump <code>@​actions/cache</code> from 3.3.0 to 4.0.0
in the dependencies group by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1135">golangci/golangci-lint-action#1135</a></li>
<li>build(deps-dev): bump the dev-dependencies group with 2 updates by
<a href="https://github.com/dependabot"><code>@​dependabot</code></a> in
<a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1136">golangci/golangci-lint-action#1136</a></li>
<li>build(deps): bump <code>@​types/node</code> from 22.10.1 to 22.10.2
in the dependencies group by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1137">golangci/golangci-lint-action#1137</a></li>
<li>build(deps-dev): bump the dev-dependencies group with 2 updates by
<a href="https://github.com/dependabot"><code>@​dependabot</code></a> in
<a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1138">golangci/golangci-lint-action#1138</a></li>
<li>build(deps-dev): bump the dev-dependencies group with 2 updates by
<a href="https://github.com/dependabot"><code>@​dependabot</code></a> in
<a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1139">golangci/golangci-lint-action#1139</a></li>
<li>build(deps-dev): bump the dev-dependencies group with 2 updates by
<a href="https://github.com/dependabot"><code>@​dependabot</code></a> in
<a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1141">golangci/golangci-lint-action#1141</a></li>
<li>build(deps): bump <code>@​types/node</code> from 22.10.2 to 22.10.5
in the dependencies group by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1142">golangci/golangci-lint-action#1142</a></li>
<li>build(deps-dev): bump the dev-dependencies group with 3 updates by
<a href="https://github.com/dependabot"><code>@​dependabot</code></a> in
<a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1143">golangci/golangci-lint-action#1143</a></li>
</ul>
<h2>New Contributors</h2>
<ul>
<li><a href="https://github.com/dmitris"><code>@​dmitris</code></a> made
their first contribution in <a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1125">golangci/golangci-lint-action#1125</a></li>
<li><a
href="https://github.com/alexandear"><code>@​alexandear</code></a> made
their first contribution in <a
href="https://redirect.github.com/golangci/golangci-lint-action/pull/1133">golangci/golangci-lint-action#1133</a></li>
</ul>
<p><strong>Full Changelog</strong>: <a
href="https://github.com/golangci/golangci-lint-action/compare/v6.1.1...v6.2.0">https://github.com/golangci/golangci-lint-action/compare/v6.1.1...v6.2.0</a></p>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/golangci/golangci-lint-action/commit/ec5d18412c0aeab7936cb16880d708ba2a64e1ae"><code>ec5d184</code></a>
feat: support linux arm64 public preview (<a
href="https://redirect.github.com/golangci/golangci-lint-action/issues/1144">#1144</a>)</li>
<li><a
href="https://github.com/golangci/golangci-lint-action/commit/a0297a137827338a46f13803e445426219bb07be"><code>a0297a1</code></a>
build(deps-dev): bump the dev-dependencies group with 3 updates (<a
href="https://redirect.github.com/golangci/golangci-lint-action/issues/1143">#1143</a>)</li>
<li><a
href="https://github.com/golangci/golangci-lint-action/commit/58eda26a511c265ee35b3ee4b101fb8adfd76480"><code>58eda26</code></a>
build(deps): bump <code>@​types/node</code> from 22.10.2 to 22.10.5 in
the dependencies gro...</li>
<li><a
href="https://github.com/golangci/golangci-lint-action/commit/44c24345067b52de8ba69bbc06171f3eb93ae74d"><code>44c2434</code></a>
build(deps-dev): bump the dev-dependencies group with 2 updates (<a
href="https://redirect.github.com/golangci/golangci-lint-action/issues/1141">#1141</a>)</li>
<li><a
href="https://github.com/golangci/golangci-lint-action/commit/2f13b8027d5e60ac6e32b086a0dc4d2ae4cb3f77"><code>2f13b80</code></a>
build(deps-dev): bump the dev-dependencies group with 2 updates (<a
href="https://redirect.github.com/golangci/golangci-lint-action/issues/1139">#1139</a>)</li>
<li><a
href="https://github.com/golangci/golangci-lint-action/commit/1ac36865a67f3dce6e44b851a4a8c068272a9d34"><code>1ac3686</code></a>
build(deps-dev): bump the dev-dependencies group with 2 updates (<a
href="https://redirect.github.com/golangci/golangci-lint-action/issues/1138">#1138</a>)</li>
<li><a
href="https://github.com/golangci/golangci-lint-action/commit/9937fdf7189f2958a2dc9f6d585e5d65e3326d20"><code>9937fdf</code></a>
build(deps): bump <code>@​types/node</code> from 22.10.1 to 22.10.2 in
the dependencies gro...</li>
<li><a
href="https://github.com/golangci/golangci-lint-action/commit/cb60b26e7ac8271a28750fffe50d08cd8194be4c"><code>cb60b26</code></a>
build(deps-dev): bump the dev-dependencies group with 2 updates (<a
href="https://redirect.github.com/golangci/golangci-lint-action/issues/1136">#1136</a>)</li>
<li><a
href="https://github.com/golangci/golangci-lint-action/commit/774c35bcccffb734694af9e921f12f57d882ef74"><code>774c35b</code></a>
build(deps): bump <code>@​actions/cache</code> from 3.3.0 to 4.0.0 in
the dependencies grou...</li>
<li><a
href="https://github.com/golangci/golangci-lint-action/commit/7ce548721e510362ce7ad242d926464152b47c18"><code>7ce5487</code></a>
build(deps-dev): bump the dev-dependencies group with 3 updates (<a
href="https://redirect.github.com/golangci/golangci-lint-action/issues/1134">#1134</a>)</li>
<li>Additional commits viewable in <a
href="https://github.com/golangci/golangci-lint-action/compare/971e284b6050e8a5849b72094c50ab08da042db8...ec5d18412c0aeab7936cb16880d708ba2a64e1ae">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golangci/golangci-lint-action&package-manager=github_actions&previous-version=6.1.1&new-version=6.2.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/push.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
index 2a8a68862..5921b4e5f 100644
--- a/.github/workflows/push.yml
+++ b/.github/workflows/push.yml
@@ -95,7 +95,7 @@ jobs:
           # Exit with status code 1 if there are differences (i.e. unformatted files)
           git diff --exit-code
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1
+        uses: golangci/golangci-lint-action@ec5d18412c0aeab7936cb16880d708ba2a64e1ae # v6.2.0
         with:
           version: v1.63.4
           args: --timeout=15m

From 75932198f72a2e0e658d36148b539ef545e8d828 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 3 Feb 2025 12:36:42 +0100
Subject: [PATCH 186/247] Bump astral-sh/ruff-action from 3.0.1 to 3.1.0
 (#2274)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bumps [astral-sh/ruff-action](https://github.com/astral-sh/ruff-action)
from 3.0.1 to 3.1.0.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/astral-sh/ruff-action/releases">astral-sh/ruff-action's
releases</a>.</em></p>
<blockquote>
<h2>v3.1.0 🌈 Determine ruff version from optional or dependency
groups</h2>
<h2>Changes</h2>
<p>Big thank you to <a
href="https://github.com/AA-Turner"><code>@​AA-Turner</code></a> for
expanding the pyproject.toml parsing to also find the ruff version to
use in the following scenarios:</p>
<pre lang="toml"><code>[dependency-groups]
dev = [
    { include-group = &quot;docs&quot; },
    { include-group = &quot;lint&quot; },
]
docs = [
    &quot;sphinx&quot;,
]
lint = [
    &quot;ruff==0.8.3&quot;,
]
</code></pre>
<pre lang="toml"><code>[project.optional-dependencies]
lint = [
    &quot;ruff==0.8.3&quot;,
]
</code></pre>
<h2>🚀 Enhancements</h2>
<ul>
<li>Read the <code>[project.optional-dependencies]</code> and
<code>[dependency-groups]</code> tables <a
href="https://github.com/AA-Turner"><code>@​AA-Turner</code></a> (<a
href="https://redirect.github.com/astral-sh/ruff-action/issues/66">#66</a>)</li>
</ul>
<h2>v3.0.2 🌈 Full support for GHES</h2>
<h2>Changes</h2>
<p>This release fixes some issues that prevented use with GitHub
Enterprise Server instances.
Parsing the ruff version from pyproject.toml now also uses a library
that is fully TOML 1.0.0 compliant.</p>
<h2>🐛 Bug fixes</h2>
<ul>
<li>Do not expect GITHUB_TOKEN to be set or valid <a
href="https://github.com/eifinger"><code>@​eifinger</code></a> (<a
href="https://redirect.github.com/astral-sh/ruff-action/issues/65">#65</a>)</li>
<li>Use TOML 1.0.0 compliant library for parsing <a
href="https://github.com/eifinger"><code>@​eifinger</code></a> (<a
href="https://redirect.github.com/astral-sh/ruff-action/issues/47">#47</a>)</li>
</ul>
<h2>🧰 Maintenance</h2>
<ul>
<li>Fix compiled known versions <a
href="https://github.com/eifinger"><code>@​eifinger</code></a> (<a
href="https://redirect.github.com/astral-sh/ruff-action/issues/62">#62</a>)</li>
<li>chore: update known checksums for 0.9.3 @<a
href="https://github.com/apps/github-actions">github-actions[bot]</a>
(<a
href="https://redirect.github.com/astral-sh/ruff-action/issues/61">#61</a>)</li>
<li>chore: update known checksums for 0.9.1 @<a
href="https://github.com/apps/github-actions">github-actions[bot]</a>
(<a
href="https://redirect.github.com/astral-sh/ruff-action/issues/42">#42</a>)</li>
</ul>
<h2>📚 Documentation</h2>
<ul>
<li>Fix Markdown link to Install the latest version <a
href="https://github.com/eifinger"><code>@​eifinger</code></a> (<a
href="https://redirect.github.com/astral-sh/ruff-action/issues/58">#58</a>)</li>
</ul>
<!-- raw HTML omitted -->
</blockquote>
<p>... (truncated)</p>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/astral-sh/ruff-action/commit/f14634c415d3e63ffd4d550a22f037df4c734a60"><code>f14634c</code></a>
Read the <code>[project.optional-dependencies]</code> and
<code>[dependency-groups]</code> tables (...</li>
<li><a
href="https://github.com/astral-sh/ruff-action/commit/47de3deae877b8fcf76756bc14e8f8edd5e03e7d"><code>47de3de</code></a>
Bump <code>@​types/node</code> from 22.10.10 to 22.12.0 (<a
href="https://redirect.github.com/astral-sh/ruff-action/issues/60">#60</a>)</li>
<li><a
href="https://github.com/astral-sh/ruff-action/commit/d8281c74d46e369a96801b98fb140e53aaba5432"><code>d8281c7</code></a>
Do not expect GITHUB_TOKEN to be set or valid (<a
href="https://redirect.github.com/astral-sh/ruff-action/issues/65">#65</a>)</li>
<li><a
href="https://github.com/astral-sh/ruff-action/commit/a6340446592efa6ab1e62e3c62d19e1ceaf0d4bf"><code>a634044</code></a>
Bump eifinger/actionlint-action from 1.9.0 to 1.9.1 (<a
href="https://redirect.github.com/astral-sh/ruff-action/issues/59">#59</a>)</li>
<li><a
href="https://github.com/astral-sh/ruff-action/commit/2993ff4a656ec52e8aa9463021afad39beeebd31"><code>2993ff4</code></a>
Fix compiled known versions (<a
href="https://redirect.github.com/astral-sh/ruff-action/issues/62">#62</a>)</li>
<li><a
href="https://github.com/astral-sh/ruff-action/commit/20a3b171f495b27832b0e47aeda184f182bb2e40"><code>20a3b17</code></a>
chore: update known checksums for 0.9.3 (<a
href="https://redirect.github.com/astral-sh/ruff-action/issues/61">#61</a>)</li>
<li><a
href="https://github.com/astral-sh/ruff-action/commit/1c1aef9e3df06779da107441bd827774ed2d315b"><code>1c1aef9</code></a>
Bump typescript from 5.7.2 to 5.7.3 (<a
href="https://redirect.github.com/astral-sh/ruff-action/issues/41">#41</a>)</li>
<li><a
href="https://github.com/astral-sh/ruff-action/commit/0ceb04d9a0c11be190af496dccc164aff1366592"><code>0ceb04d</code></a>
Bump release-drafter/release-drafter from 6.0.0 to 6.1.0 (<a
href="https://redirect.github.com/astral-sh/ruff-action/issues/50">#50</a>)</li>
<li><a
href="https://github.com/astral-sh/ruff-action/commit/18db80c95405162b0ccba8f553e2d103bdc2bac9"><code>18db80c</code></a>
Bump <code>@​types/node</code> from 22.10.5 to 22.10.10 (<a
href="https://redirect.github.com/astral-sh/ruff-action/issues/53">#53</a>)</li>
<li><a
href="https://github.com/astral-sh/ruff-action/commit/0a5dfb89f1b5a1ecfece8802e3948a3c91281fbe"><code>0a5dfb8</code></a>
Fix Markdown link to Install the latest version (<a
href="https://redirect.github.com/astral-sh/ruff-action/issues/58">#58</a>)</li>
<li>Additional commits viewable in <a
href="https://github.com/astral-sh/ruff-action/compare/31a518504640beb4897d0b9f9e50a2a9196e75ba...f14634c415d3e63ffd4d550a22f037df4c734a60">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=astral-sh/ruff-action&package-manager=github_actions&previous-version=3.0.1&new-version=3.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/push.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
index 5921b4e5f..f27459baa 100644
--- a/.github/workflows/push.yml
+++ b/.github/workflows/push.yml
@@ -100,7 +100,7 @@ jobs:
           version: v1.63.4
           args: --timeout=15m
       - name: Run ruff
-        uses: astral-sh/ruff-action@31a518504640beb4897d0b9f9e50a2a9196e75ba # v3.0.1
+        uses: astral-sh/ruff-action@f14634c415d3e63ffd4d550a22f037df4c734a60 # v3.1.0
         with:
           version: "0.9.1"
           args: "format --check"

From 4f3a289333094951fe7b55b00a7f3bc70cd138eb Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 3 Feb 2025 12:36:50 +0100
Subject: [PATCH 187/247] Bump actions/stale from 9.0.0 to 9.1.0 (#2275)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bumps [actions/stale](https://github.com/actions/stale) from 9.0.0 to
9.1.0.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/actions/stale/releases">actions/stale's
releases</a>.</em></p>
<blockquote>
<h2>v9.1.0</h2>
<h2>What's Changed</h2>
<ul>
<li>Documentation update by <a
href="https://github.com/Marukome0743"><code>@​Marukome0743</code></a>
in <a
href="https://redirect.github.com/actions/stale/pull/1116">actions/stale#1116</a></li>
<li>Add workflow file for publishing releases to immutable action
package by <a
href="https://github.com/Jcambass"><code>@​Jcambass</code></a> in <a
href="https://redirect.github.com/actions/stale/pull/1179">actions/stale#1179</a></li>
<li>Update undici from 5.28.2 to 5.28.4 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/actions/stale/pull/1150">actions/stale#1150</a></li>
<li>Update actions/checkout from 3 to 4 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/actions/stale/pull/1091">actions/stale#1091</a></li>
<li>Update actions/publish-action from 0.2.2 to 0.3.0 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/actions/stale/pull/1147">actions/stale#1147</a></li>
<li>Update ts-jest from 29.1.1 to 29.2.5 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/actions/stale/pull/1175">actions/stale#1175</a></li>
<li>Update <code>@​actions/core</code> from 1.10.1 to 1.11.1 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/actions/stale/pull/1191">actions/stale#1191</a></li>
<li>Update <code>@​types/jest</code> from 29.5.11 to 29.5.14 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/actions/stale/pull/1193">actions/stale#1193</a></li>
<li>Update <code>@​actions/cache</code> from 3.2.2 to 4.0.0 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/actions/stale/pull/1194">actions/stale#1194</a></li>
</ul>
<h2>New Contributors</h2>
<ul>
<li><a
href="https://github.com/Marukome0743"><code>@​Marukome0743</code></a>
made their first contribution in <a
href="https://redirect.github.com/actions/stale/pull/1116">actions/stale#1116</a></li>
<li><a href="https://github.com/Jcambass"><code>@​Jcambass</code></a>
made their first contribution in <a
href="https://redirect.github.com/actions/stale/pull/1179">actions/stale#1179</a></li>
</ul>
<p><strong>Full Changelog</strong>: <a
href="https://github.com/actions/stale/compare/v9...v9.1.0">https://github.com/actions/stale/compare/v9...v9.1.0</a></p>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/actions/stale/commit/5bef64f19d7facfb25b37b414482c7164d639639"><code>5bef64f</code></a>
build(deps): bump <code>@​actions/cache</code> from 3.2.2 to 4.0.0 (<a
href="https://redirect.github.com/actions/stale/issues/1194">#1194</a>)</li>
<li><a
href="https://github.com/actions/stale/commit/fa77dfddd04682b7d96dbc4e016318e681fdc10e"><code>fa77dfd</code></a>
build(deps-dev): bump <code>@​types/jest</code> from 29.5.11 to 29.5.14
(<a
href="https://redirect.github.com/actions/stale/issues/1193">#1193</a>)</li>
<li><a
href="https://github.com/actions/stale/commit/f04443dce335c74ba15c65f4cbb3688e6cb6a6ec"><code>f04443d</code></a>
build(deps): bump <code>@​actions/core</code> from 1.10.1 to 1.11.1 (<a
href="https://redirect.github.com/actions/stale/issues/1191">#1191</a>)</li>
<li><a
href="https://github.com/actions/stale/commit/5c715b0513651880806e14d529f014b12fdd50eb"><code>5c715b0</code></a>
build(deps-dev): bump ts-jest from 29.1.1 to 29.2.5 (<a
href="https://redirect.github.com/actions/stale/issues/1175">#1175</a>)</li>
<li><a
href="https://github.com/actions/stale/commit/f69122271d990fd11f5594ccff2296f00ff59b49"><code>f691222</code></a>
build(deps): bump actions/publish-action from 0.2.2 to 0.3.0 (<a
href="https://redirect.github.com/actions/stale/issues/1147">#1147</a>)</li>
<li><a
href="https://github.com/actions/stale/commit/df990c2cf5ae92c90653c9485d6882a0a09feac7"><code>df990c2</code></a>
build(deps): bump actions/checkout from 3 to 4 (<a
href="https://redirect.github.com/actions/stale/issues/1091">#1091</a>)</li>
<li><a
href="https://github.com/actions/stale/commit/6e472ce44ab4197b0154601c59c54a75b73b340b"><code>6e472ce</code></a>
Merge pull request <a
href="https://redirect.github.com/actions/stale/issues/1179">#1179</a>
from actions/Jcambass-patch-1</li>
<li><a
href="https://github.com/actions/stale/commit/d10ba64261d965f75165f74c55cd3ffbf690d442"><code>d10ba64</code></a>
Merge pull request <a
href="https://redirect.github.com/actions/stale/issues/1150">#1150</a>
from actions/dependabot/npm_and_yarn/undici-5.28.4</li>
<li><a
href="https://github.com/actions/stale/commit/bbf3da5f64eebd003932d93293857400f7f7e18d"><code>bbf3da5</code></a>
resolve check failures</li>
<li><a
href="https://github.com/actions/stale/commit/6a2e61d18b155e538f85ef1bf7bd0470775e9703"><code>6a2e61d</code></a>
Add workflow file for publishing releases to immutable action
package</li>
<li>Additional commits viewable in <a
href="https://github.com/actions/stale/compare/28ca1036281a5e5922ead5184a1bbf96e5fc984e...5bef64f19d7facfb25b37b414482c7164d639639">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/stale&package-manager=github_actions&previous-version=9.0.0&new-version=9.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/close-stale-issues.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/close-stale-issues.yml b/.github/workflows/close-stale-issues.yml
index ea9558caf..fc764fb0d 100644
--- a/.github/workflows/close-stale-issues.yml
+++ b/.github/workflows/close-stale-issues.yml
@@ -18,7 +18,7 @@ jobs:
       pull-requests: write
 
     steps:
-      - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0
+      - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0
         with:
           stale-issue-message: This issue has not received a response in a while. If you want to keep this issue open, please leave a comment below and auto-close will be canceled.
           stale-pr-message: This PR has not received an update in a while. If you want to keep this PR open, please leave a comment below or push a new commit and auto-close will be canceled.

From 75db82ae1f5f3fc12716cb7441d0374121dcf6b4 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 3 Feb 2025 12:37:17 +0100
Subject: [PATCH 188/247] Bump actions/create-github-app-token from 1.11.1 to
 1.11.2 (#2276)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bumps
[actions/create-github-app-token](https://github.com/actions/create-github-app-token)
from 1.11.1 to 1.11.2.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/actions/create-github-app-token/releases">actions/create-github-app-token's
releases</a>.</em></p>
<blockquote>
<h2>v1.11.2</h2>
<h2><a
href="https://github.com/actions/create-github-app-token/compare/v1.11.1...v1.11.2">1.11.2</a>
(2025-01-30)</h2>
<h3>Bug Fixes</h3>
<ul>
<li><strong>deps:</strong> bump <code>@​octokit/request</code> from
9.1.3 to 9.1.4 in the production-dependencies group (<a
href="https://redirect.github.com/actions/create-github-app-token/issues/196">#196</a>)
(<a
href="https://github.com/actions/create-github-app-token/commit/b4192a5b3659c0e5875f868c44727ef4ecc5d579">b4192a5</a>),
closes <a
href="https://redirect.github.com/actions/create-github-app-token/issues/730">#730</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/730">#730</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/729">#729</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/727">#727</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/726">#726</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/723">#723</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/724">#724</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/722">#722</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/721">#721</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/720">#720</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/719">#719</a></li>
<li><strong>deps:</strong> bump undici from 6.19.8 to 7.2.0 (<a
href="https://redirect.github.com/actions/create-github-app-token/issues/198">#198</a>)
(<a
href="https://github.com/actions/create-github-app-token/commit/29aa0514a79f3b4aa8b1547173a41455949e4bf6">29aa051</a>),
closes <a
href="https://redirect.github.com/nodejs/undici/issues/3958">nodejs/undici#3958</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3955">nodejs/undici#3955</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3962">nodejs/undici#3962</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3921">nodejs/undici#3921</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3923">nodejs/undici#3923</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3925">nodejs/undici#3925</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3926">nodejs/undici#3926</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3924">nodejs/undici#3924</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3933">nodejs/undici#3933</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3916">nodejs/undici#3916</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3930">nodejs/undici#3930</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3938">nodejs/undici#3938</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/3937">#3937</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3940">nodejs/undici#3940</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3931">nodejs/undici#3931</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3941">nodejs/undici#3941</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3911">nodejs/undici#3911</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3888">nodejs/undici#3888</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3939">nodejs/undici#3939</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3947">nodejs/undici#3947</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3945">nodejs/undici#3945</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3916">nodejs/undici#3916</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3893">nodejs/undici#3893</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3902">nodejs/undici#3902</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/3901">#3901</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3903">nodejs/undici#3903</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3905">nodejs/undici#3905</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3900">nodejs/undici#3900</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3913">nodejs/undici#3913</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3910">nodejs/undici#3910</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3909">nodejs/undici#3909</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3906">nodejs/undici#3906</a>
<a
href="https://redirect.github.com/nodejs/undici/issues/3922">nodejs/undici#3922</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/3962">#3962</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/3955">#3955</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/3958">#3958</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/3945">#3945</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/3947">#3947</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/3939">#3939</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/3888">#3888</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/3911">#3911</a>
<a
href="https://redirect.github.com/actions/create-github-app-token/issues/3941">#3941</a></li>
</ul>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/actions/create-github-app-token/commit/136412a57a7081aa63c935a2cc2918f76c34f514"><code>136412a</code></a>
build(release): 1.11.2 [skip ci]</li>
<li><a
href="https://github.com/actions/create-github-app-token/commit/b4192a5b3659c0e5875f868c44727ef4ecc5d579"><code>b4192a5</code></a>
fix(deps): bump <code>@​octokit/request</code> from 9.1.3 to 9.1.4 in
the production-depend...</li>
<li><a
href="https://github.com/actions/create-github-app-token/commit/29aa0514a79f3b4aa8b1547173a41455949e4bf6"><code>29aa051</code></a>
fix(deps): bump undici from 6.19.8 to 7.2.0 (<a
href="https://redirect.github.com/actions/create-github-app-token/issues/198">#198</a>)</li>
<li><a
href="https://github.com/actions/create-github-app-token/commit/a5f8600f5881d59ee9299dfd57605d0ccbd561f5"><code>a5f8600</code></a>
build(deps-dev): bump <code>@​sinonjs/fake-timers</code> from 13.0.2 to
14.0.0 (<a
href="https://redirect.github.com/actions/create-github-app-token/issues/199">#199</a>)</li>
<li><a
href="https://github.com/actions/create-github-app-token/commit/0edddd70c80d313f6d5fac3b29df92440abb9e33"><code>0edddd7</code></a>
build(deps-dev): bump the development-dependencies group with 2 updates
(<a
href="https://redirect.github.com/actions/create-github-app-token/issues/197">#197</a>)</li>
<li><a
href="https://github.com/actions/create-github-app-token/commit/bb3ca765afdde129dbde9599979c21f6d97e9b0a"><code>bb3ca76</code></a>
docs(README): remove extra space in variable syntax in README example
(<a
href="https://redirect.github.com/actions/create-github-app-token/issues/201">#201</a>)</li>
<li>See full diff in <a
href="https://github.com/actions/create-github-app-token/compare/c1a285145b9d317df6ced56c09f525b5c2b6f755...136412a57a7081aa63c935a2cc2918f76c34f514">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/create-github-app-token&package-manager=github_actions&previous-version=1.11.1&new-version=1.11.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/integration-main.yml | 2 +-
 .github/workflows/integration-pr.yml   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/integration-main.yml b/.github/workflows/integration-main.yml
index 84dd7263a..f737c48e6 100644
--- a/.github/workflows/integration-main.yml
+++ b/.github/workflows/integration-main.yml
@@ -20,7 +20,7 @@ jobs:
     steps:
       - name: Generate GitHub App Token
         id: generate-token
-        uses: actions/create-github-app-token@c1a285145b9d317df6ced56c09f525b5c2b6f755 # v1.11.1
+        uses: actions/create-github-app-token@136412a57a7081aa63c935a2cc2918f76c34f514 # v1.11.2
         with:
           app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}
           private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }}
diff --git a/.github/workflows/integration-pr.yml b/.github/workflows/integration-pr.yml
index 7a62113cd..bf096c863 100644
--- a/.github/workflows/integration-pr.yml
+++ b/.github/workflows/integration-pr.yml
@@ -23,7 +23,7 @@ jobs:
     steps:
       - name: Generate GitHub App Token
         id: generate-token
-        uses: actions/create-github-app-token@c1a285145b9d317df6ced56c09f525b5c2b6f755 # v1.11.1
+        uses: actions/create-github-app-token@136412a57a7081aa63c935a2cc2918f76c34f514 # v1.11.2
         with:
           app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}
           private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }}

From 838de2fde23b2234843671ff38f021d1356edf67 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 3 Feb 2025 12:38:16 +0100
Subject: [PATCH 189/247] Bump github.com/hashicorp/terraform-exec from 0.21.0
 to 0.22.0 (#2237)

Bumps
[github.com/hashicorp/terraform-exec](https://github.com/hashicorp/terraform-exec)
from 0.21.0 to 0.22.0.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/hashicorp/terraform-exec/releases">github.com/hashicorp/terraform-exec's
releases</a>.</em></p>
<blockquote>
<h2>v0.22.0</h2>
<p>ENHANCEMENTS:</p>
<ul>
<li>tfexec: Add support for <code>terraform init --json</code> via
<code>InitJSON</code> (<a
href="https://redirect.github.com/hashicorp/terraform-exec/pull/478">#478</a>)</li>
</ul>
<p>INTERNAL:</p>
<ul>
<li>go: Require Go 1.22 (previously 1.18) (<a
href="https://redirect.github.com/hashicorp/terraform-exec/pull/499">#499</a>)</li>
</ul>
</blockquote>
</details>
<details>
<summary>Changelog</summary>
<p><em>Sourced from <a
href="https://github.com/hashicorp/terraform-exec/blob/main/CHANGELOG.md">github.com/hashicorp/terraform-exec's
changelog</a>.</em></p>
<blockquote>
<h1>0.22.0 (January 21, 2025)</h1>
<p>ENHANCEMENTS:</p>
<ul>
<li>tfexec: Add support for <code>terraform init --json</code> via
<code>InitJSON</code> (<a
href="https://redirect.github.com/hashicorp/terraform-exec/pull/478">#478</a>)</li>
</ul>
<p>INTERNAL:</p>
<ul>
<li>go: Require Go 1.22 (previously 1.18) (<a
href="https://redirect.github.com/hashicorp/terraform-exec/pull/499">#499</a>)</li>
</ul>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/6801a6e6fd18d611c96e1d3fed1641979a8bb978"><code>6801a6e</code></a>
v0.22.0 [skip ci]</li>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/dd2bc9a8a9805fd60743db96fe1d85373861b7d5"><code>dd2bc9a</code></a>
Update CHANGELOG.md (<a
href="https://redirect.github.com/hashicorp/terraform-exec/issues/501">#501</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/b5e5740e8dc725ec9b01e75c6f8eca65cd57d57e"><code>b5e5740</code></a>
build(deps): bump github.com/hashicorp/hc-install from 0.8.0 to 0.9.1
(<a
href="https://redirect.github.com/hashicorp/terraform-exec/issues/494">#494</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/abfb5ba96eac8a5a4e7952d33d8ab5f3f9a76ae9"><code>abfb5ba</code></a>
tfexec: add InitJSON (<a
href="https://redirect.github.com/hashicorp/terraform-exec/issues/478">#478</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/840ecadf662494cfd164198427b433f98782ae36"><code>840ecad</code></a>
ci/e2etests: Add latest major Terraform versions (<a
href="https://redirect.github.com/hashicorp/terraform-exec/issues/498">#498</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/4497f9edf36dcdbbf5fc982d1d951c9509d865fd"><code>4497f9e</code></a>
go: Require Go 1.22 (previously 1.18) (<a
href="https://redirect.github.com/hashicorp/terraform-exec/issues/499">#499</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/b13b10be14c209dbd1137cb02700c0e9b488a324"><code>b13b10b</code></a>
build(deps): bump github.com/zclconf/go-cty from 1.16.0 to 1.16.1 (<a
href="https://redirect.github.com/hashicorp/terraform-exec/issues/496">#496</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/6b0d5eb88aac954ad4e7b032472bd89cbe9539f1"><code>6b0d5eb</code></a>
build(deps): bump github.com/zclconf/go-cty from 1.15.1 to 1.16.0 (<a
href="https://redirect.github.com/hashicorp/terraform-exec/issues/495">#495</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/ef0b6c386e20117f112ab928f511dc9f66d46c4d"><code>ef0b6c3</code></a>
build(deps): Bump workflows to latest trusted versions (<a
href="https://redirect.github.com/hashicorp/terraform-exec/issues/493">#493</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/c75d998af596d532c54b807801b52ee743af3871"><code>c75d998</code></a>
build(deps): bump github.com/hashicorp/terraform-json from 0.23.0 to
0.24.0 (...</li>
<li>Additional commits viewable in <a
href="https://github.com/hashicorp/terraform-exec/compare/v0.21.0...v0.22.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/hashicorp/terraform-exec&package-manager=go_modules&previous-version=0.21.0&new-version=0.22.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod |  6 +++---
 go.sum | 12 ++++++------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/go.mod b/go.mod
index bd8997190..151133944 100644
--- a/go.mod
+++ b/go.mod
@@ -13,8 +13,8 @@ require (
 	github.com/google/uuid v1.6.0 // BSD-3-Clause
 	github.com/hashicorp/go-version v1.7.0 // MPL 2.0
 	github.com/hashicorp/hc-install v0.9.1 // MPL 2.0
-	github.com/hashicorp/terraform-exec v0.21.0 // MPL 2.0
-	github.com/hashicorp/terraform-json v0.23.0 // MPL 2.0
+	github.com/hashicorp/terraform-exec v0.22.0 // MPL 2.0
+	github.com/hashicorp/terraform-json v0.24.0 // MPL 2.0
 	github.com/hexops/gotextdiff v1.0.3 // BSD 3-Clause "New" or "Revised" License
 	github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause
 	github.com/mattn/go-isatty v0.0.20 // MIT
@@ -62,7 +62,7 @@ require (
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tidwall/sjson v1.2.5 // indirect
-	github.com/zclconf/go-cty v1.15.0 // indirect
+	github.com/zclconf/go-cty v1.16.1 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
 	go.opentelemetry.io/otel v1.24.0 // indirect
diff --git a/go.sum b/go.sum
index dec1d40b2..3c7f20937 100644
--- a/go.sum
+++ b/go.sum
@@ -107,10 +107,10 @@ github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKe
 github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/hc-install v0.9.1 h1:gkqTfE3vVbafGQo6VZXcy2v5yoz2bE0+nhZXruCuODQ=
 github.com/hashicorp/hc-install v0.9.1/go.mod h1:pWWvN/IrfeBK4XPeXXYkL6EjMufHkCK5DvwxeLKuBf0=
-github.com/hashicorp/terraform-exec v0.21.0 h1:uNkLAe95ey5Uux6KJdua6+cv8asgILFVWkd/RG0D2XQ=
-github.com/hashicorp/terraform-exec v0.21.0/go.mod h1:1PPeMYou+KDUSSeRE9szMZ/oHf4fYUmB923Wzbq1ICg=
-github.com/hashicorp/terraform-json v0.23.0 h1:sniCkExU4iKtTADReHzACkk8fnpQXrdD2xoR+lppBkI=
-github.com/hashicorp/terraform-json v0.23.0/go.mod h1:MHdXbBAbSg0GvzuWazEGKAn/cyNfIB7mN6y7KJN6y2c=
+github.com/hashicorp/terraform-exec v0.22.0 h1:G5+4Sz6jYZfRYUCg6eQgDsqTzkNXV+fP8l+uRmZHj64=
+github.com/hashicorp/terraform-exec v0.22.0/go.mod h1:bjVbsncaeh8jVdhttWYZuBGj21FcYw6Ia/XfHcNO7lQ=
+github.com/hashicorp/terraform-json v0.24.0 h1:rUiyF+x1kYawXeRth6fKFm/MdfBS6+lW4NbeATsYz8Q=
+github.com/hashicorp/terraform-json v0.24.0/go.mod h1:Nfj5ubo9xbu9uiAoZVBsNOjvNKB66Oyrvtit74kC7ow=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
@@ -174,8 +174,8 @@ github.com/wI2L/jsondiff v0.6.1 h1:ISZb9oNWbP64LHnu4AUhsMF5W0FIj5Ok3Krip9Shqpw=
 github.com/wI2L/jsondiff v0.6.1/go.mod h1:KAEIojdQq66oJiHhDyQez2x+sRit0vIzC9KeK0yizxM=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
-github.com/zclconf/go-cty v1.15.0 h1:tTCRWxsexYUmtt/wVxgDClUe+uQusuI443uL6e+5sXQ=
-github.com/zclconf/go-cty v1.15.0/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
+github.com/zclconf/go-cty v1.16.1 h1:a5TZEPzBFFR53udlIKApXzj8JIF4ZNQ6abH79z5R1S0=
+github.com/zclconf/go-cty v1.16.1/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg=

From 9320bd1682a22df57950765d71d21b44852084a1 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 3 Feb 2025 15:10:19 +0100
Subject: [PATCH 190/247] acc: Use [VARNAME] instead of $VARNAME (#2282)

$VARNAME is what we use for environment variables, it's good to
separate.

Some people use envsubst for homemade variable interpolation, it's also
good to have separation there.
---
 acceptance/acceptance_test.go                 | 12 ++--
 acceptance/bundle/git-permerror/output.txt    | 18 +++---
 .../bundle/help/bundle-deploy/output.txt      |  2 +-
 .../bundle/help/bundle-deployment/output.txt  |  2 +-
 .../bundle/help/bundle-destroy/output.txt     |  2 +-
 .../help/bundle-generate-dashboard/output.txt |  2 +-
 .../help/bundle-generate-job/output.txt       |  2 +-
 .../help/bundle-generate-pipeline/output.txt  |  2 +-
 .../bundle/help/bundle-generate/output.txt    |  2 +-
 acceptance/bundle/help/bundle-init/output.txt |  2 +-
 acceptance/bundle/help/bundle-open/output.txt |  2 +-
 acceptance/bundle/help/bundle-run/output.txt  |  2 +-
 .../bundle/help/bundle-schema/output.txt      |  2 +-
 .../bundle/help/bundle-summary/output.txt     |  2 +-
 acceptance/bundle/help/bundle-sync/output.txt |  2 +-
 .../bundle/help/bundle-validate/output.txt    |  2 +-
 acceptance/bundle/help/bundle/output.txt      |  2 +-
 .../bundle/override/clusters/output.txt       |  4 +-
 .../bundle/override/job_cluster/output.txt    |  8 +--
 .../override/job_cluster_var/output.txt       | 20 +++----
 .../job_tasks/out.development.stderr.txt      |  2 +-
 .../bundle/override/job_tasks/output.txt      |  8 +--
 .../override/merge-string-map/output.txt      |  8 +--
 .../override/pipeline_cluster/output.txt      |  8 +--
 .../bundle/paths/fallback/output.job.json     |  8 +--
 .../paths/fallback/output.pipeline.json       |  8 +--
 acceptance/bundle/paths/fallback/output.txt   |  8 +--
 .../bundle/paths/nominal/output.job.json      | 12 ++--
 .../bundle/paths/nominal/output.pipeline.json |  8 +--
 acceptance/bundle/paths/nominal/output.txt    |  8 +--
 .../relative_path_translation/output.txt      |  4 +-
 acceptance/bundle/quality_monitor/output.txt  |  6 +-
 acceptance/bundle/scripts/output.txt          | 12 ++--
 .../bundle/syncroot/dotdot-git/output.txt     |  6 +-
 .../bundle/syncroot/dotdot-nogit/output.txt   |  4 +-
 .../bundle/templates/dbt-sql/output.txt       | 20 +++----
 .../dbt-sql/output/my_dbt_sql/databricks.yml  | 12 ++--
 .../output/my_dbt_sql/profile_template.yml    |  4 +-
 .../my_dbt_sql/resources/my_dbt_sql.job.yml   |  2 +-
 .../templates/default-python/output.txt       | 20 +++----
 .../output/my_default_python/databricks.yml   | 12 ++--
 .../resources/my_default_python.job.yml       |  2 +-
 .../output/my_default_python/setup.py         |  2 +-
 .../bundle/templates/default-sql/output.txt   | 20 +++----
 .../output/my_default_sql/databricks.yml      | 12 ++--
 .../resources/my_default_sql_sql.job.yml      |  2 +-
 .../experimental-jobs-as-code/output.txt      | 18 +++---
 .../output/my_jobs_as_code/databricks.yml     | 12 ++--
 .../resources/my_jobs_as_code_job.py          |  2 +-
 .../bundle/templates/wrong-url/output.txt     |  2 +-
 .../bundle/variables/arg-repeat/output.txt    |  4 +-
 .../variables/complex-cycle-self/output.txt   |  4 +-
 .../bundle/variables/complex-cycle/output.txt |  4 +-
 .../bundle/variables/complex/out.default.json |  2 +-
 .../bundle/variables/complex/out.dev.json     |  2 +-
 .../bundle/variables/complex/output.txt       |  4 +-
 .../complex_multiple_files/output.txt         |  2 +-
 .../variables/double_underscore/output.txt    |  2 +-
 acceptance/bundle/variables/empty/output.txt  |  4 +-
 .../bundle/variables/env_overrides/output.txt | 16 ++---
 .../bundle/variables/file-defaults/output.txt | 22 +++----
 .../bundle/variables/file-defaults/test.toml  |  8 +--
 .../bundle/variables/git-branch/output.txt    | 40 ++++++-------
 acceptance/bundle/variables/host/output.txt   |  4 +-
 .../prepend-workspace-var/output.txt          | 20 +++----
 .../variables/resolve-nonstrings/output.txt   |  2 +-
 .../bundle/variables/vanilla/output.txt       |  8 +--
 .../variable_overrides_in_target/output.txt   | 16 ++---
 acceptance/selftest/output.txt                |  8 +--
 acceptance/selftest/test.toml                 |  4 +-
 acceptance/terraform/output.txt               |  8 +--
 acceptance/workspace/jobs/create/output.txt   |  2 +-
 .../bundle/testdata/apps/bundle_deploy.txt    |  2 +-
 .../bundle/testdata/apps/bundle_validate.txt  |  4 +-
 .../testdata/default_python/bundle_deploy.txt |  2 +-
 .../testdata/default_python/bundle_init.txt   |  2 +-
 .../default_python/bundle_summary.txt         | 58 +++++++++----------
 .../default_python/bundle_validate.txt        |  6 +-
 libs/testdiff/replacement.go                  | 49 ++++++++--------
 79 files changed, 324 insertions(+), 329 deletions(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index ccdf74bcb..871b8bd62 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -102,13 +102,13 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	}
 
 	t.Setenv("CLI", execPath)
-	repls.SetPath(execPath, "$CLI")
+	repls.SetPath(execPath, "[CLI]")
 
 	// Make helper scripts available
 	t.Setenv("PATH", fmt.Sprintf("%s%c%s", filepath.Join(cwd, "bin"), os.PathListSeparator, os.Getenv("PATH")))
 
 	tempHomeDir := t.TempDir()
-	repls.SetPath(tempHomeDir, "$TMPHOME")
+	repls.SetPath(tempHomeDir, "[TMPHOME]")
 	t.Logf("$TMPHOME=%v", tempHomeDir)
 
 	// Make use of uv cache; since we set HomeEnvVar to temporary directory, it is not picked up automatically
@@ -133,7 +133,7 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	terraformrcPath := filepath.Join(buildDir, ".terraformrc")
 	t.Setenv("TF_CLI_CONFIG_FILE", terraformrcPath)
 	t.Setenv("DATABRICKS_TF_CLI_CONFIG_FILE", terraformrcPath)
-	repls.SetPath(terraformrcPath, "$DATABRICKS_TF_CLI_CONFIG_FILE")
+	repls.SetPath(terraformrcPath, "[DATABRICKS_TF_CLI_CONFIG_FILE]")
 
 	terraformExecPath := filepath.Join(buildDir, "terraform")
 	if runtime.GOOS == "windows" {
@@ -141,10 +141,10 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	}
 	t.Setenv("DATABRICKS_TF_EXEC_PATH", terraformExecPath)
 	t.Setenv("TERRAFORM", terraformExecPath)
-	repls.SetPath(terraformExecPath, "$TERRAFORM")
+	repls.SetPath(terraformExecPath, "[TERRAFORM]")
 
 	// do it last so that full paths match first:
-	repls.SetPath(buildDir, "$BUILD_DIR")
+	repls.SetPath(buildDir, "[BUILD_DIR]")
 
 	workspaceClient, err := databricks.NewWorkspaceClient()
 	require.NoError(t, err)
@@ -226,7 +226,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		tmpDir = t.TempDir()
 	}
 
-	repls.SetPathWithParents(tmpDir, "$TMPDIR")
+	repls.SetPathWithParents(tmpDir, "[TMPDIR]")
 	repls.Repls = append(repls.Repls, config.Repls...)
 
 	scriptContents := readMergedScriptContents(t, dir)
diff --git a/acceptance/bundle/git-permerror/output.txt b/acceptance/bundle/git-permerror/output.txt
index 60e77ca0e..03ab93442 100644
--- a/acceptance/bundle/git-permerror/output.txt
+++ b/acceptance/bundle/git-permerror/output.txt
@@ -2,20 +2,20 @@
 
 >>> chmod 000 .git
 
->>> $CLI bundle validate
+>>> [CLI] bundle validate
 Error: unable to load repository specific gitconfig: open config: permission denied
 
 Name: git-permerror
 Target: default
 Workspace:
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/git-permerror/default
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/git-permerror/default
 
 Found 1 error
 
 Exit code: 1
 
->>> $CLI bundle validate -o json
+>>> [CLI] bundle validate -o json
 Error: unable to load repository specific gitconfig: open config: permission denied
 
 
@@ -24,7 +24,7 @@ Exit code: 1
   "bundle_root_path": "."
 }
 
->>> withdir subdir/a/b $CLI bundle validate -o json
+>>> withdir subdir/a/b [CLI] bundle validate -o json
 Error: unable to load repository specific gitconfig: open config: permission denied
 
 
@@ -38,12 +38,12 @@ Exit code: 1
 
 >>> chmod 000 .git/HEAD
 
->>> $CLI bundle validate -o json
+>>> [CLI] bundle validate -o json
 {
   "bundle_root_path": "."
 }
 
->>> withdir subdir/a/b $CLI bundle validate -o json
+>>> withdir subdir/a/b [CLI] bundle validate -o json
 {
   "bundle_root_path": "."
 }
@@ -53,7 +53,7 @@ Exit code: 1
 
 >>> chmod 000 .git/config
 
->>> $CLI bundle validate -o json
+>>> [CLI] bundle validate -o json
 Error: unable to load repository specific gitconfig: open config: permission denied
 
 
@@ -62,7 +62,7 @@ Exit code: 1
   "bundle_root_path": "."
 }
 
->>> withdir subdir/a/b $CLI bundle validate -o json
+>>> withdir subdir/a/b [CLI] bundle validate -o json
 Error: unable to load repository specific gitconfig: open config: permission denied
 
 
diff --git a/acceptance/bundle/help/bundle-deploy/output.txt b/acceptance/bundle/help/bundle-deploy/output.txt
index 13c903f3e..84351e375 100644
--- a/acceptance/bundle/help/bundle-deploy/output.txt
+++ b/acceptance/bundle/help/bundle-deploy/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle deploy --help
+>>> [CLI] bundle deploy --help
 Deploy bundle
 
 Usage:
diff --git a/acceptance/bundle/help/bundle-deployment/output.txt b/acceptance/bundle/help/bundle-deployment/output.txt
index ddf5b3305..4199703b3 100644
--- a/acceptance/bundle/help/bundle-deployment/output.txt
+++ b/acceptance/bundle/help/bundle-deployment/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle deployment --help
+>>> [CLI] bundle deployment --help
 Deployment related commands
 
 Usage:
diff --git a/acceptance/bundle/help/bundle-destroy/output.txt b/acceptance/bundle/help/bundle-destroy/output.txt
index d70164301..5ed9c1b7b 100644
--- a/acceptance/bundle/help/bundle-destroy/output.txt
+++ b/acceptance/bundle/help/bundle-destroy/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle destroy --help
+>>> [CLI] bundle destroy --help
 Destroy deployed bundle resources
 
 Usage:
diff --git a/acceptance/bundle/help/bundle-generate-dashboard/output.txt b/acceptance/bundle/help/bundle-generate-dashboard/output.txt
index a63ce0ff8..683175940 100644
--- a/acceptance/bundle/help/bundle-generate-dashboard/output.txt
+++ b/acceptance/bundle/help/bundle-generate-dashboard/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle generate dashboard --help
+>>> [CLI] bundle generate dashboard --help
 Generate configuration for a dashboard
 
 Usage:
diff --git a/acceptance/bundle/help/bundle-generate-job/output.txt b/acceptance/bundle/help/bundle-generate-job/output.txt
index adc3f45ae..6a4274223 100644
--- a/acceptance/bundle/help/bundle-generate-job/output.txt
+++ b/acceptance/bundle/help/bundle-generate-job/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle generate job --help
+>>> [CLI] bundle generate job --help
 Generate bundle configuration for a job
 
 Usage:
diff --git a/acceptance/bundle/help/bundle-generate-pipeline/output.txt b/acceptance/bundle/help/bundle-generate-pipeline/output.txt
index cf5f70920..05c5573b8 100644
--- a/acceptance/bundle/help/bundle-generate-pipeline/output.txt
+++ b/acceptance/bundle/help/bundle-generate-pipeline/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle generate pipeline --help
+>>> [CLI] bundle generate pipeline --help
 Generate bundle configuration for a pipeline
 
 Usage:
diff --git a/acceptance/bundle/help/bundle-generate/output.txt b/acceptance/bundle/help/bundle-generate/output.txt
index 1d77dfdbd..725f19af0 100644
--- a/acceptance/bundle/help/bundle-generate/output.txt
+++ b/acceptance/bundle/help/bundle-generate/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle generate --help
+>>> [CLI] bundle generate --help
 Generate bundle configuration
 
 Usage:
diff --git a/acceptance/bundle/help/bundle-init/output.txt b/acceptance/bundle/help/bundle-init/output.txt
index bafe5a187..fbafedea2 100644
--- a/acceptance/bundle/help/bundle-init/output.txt
+++ b/acceptance/bundle/help/bundle-init/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle init --help
+>>> [CLI] bundle init --help
 Initialize using a bundle template.
 
 TEMPLATE_PATH optionally specifies which template to use. It can be one of the following:
diff --git a/acceptance/bundle/help/bundle-open/output.txt b/acceptance/bundle/help/bundle-open/output.txt
index 8b98aa850..b8f3f118b 100644
--- a/acceptance/bundle/help/bundle-open/output.txt
+++ b/acceptance/bundle/help/bundle-open/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle open --help
+>>> [CLI] bundle open --help
 Open a resource in the browser
 
 Usage:
diff --git a/acceptance/bundle/help/bundle-run/output.txt b/acceptance/bundle/help/bundle-run/output.txt
index 17763a295..4b9efbf2a 100644
--- a/acceptance/bundle/help/bundle-run/output.txt
+++ b/acceptance/bundle/help/bundle-run/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle run --help
+>>> [CLI] bundle run --help
 Run the job or pipeline identified by KEY.
 
 The KEY is the unique identifier of the resource to run. In addition to
diff --git a/acceptance/bundle/help/bundle-schema/output.txt b/acceptance/bundle/help/bundle-schema/output.txt
index 8f2983f5b..8b8a6b8e9 100644
--- a/acceptance/bundle/help/bundle-schema/output.txt
+++ b/acceptance/bundle/help/bundle-schema/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle schema --help
+>>> [CLI] bundle schema --help
 Generate JSON Schema for bundle configuration
 
 Usage:
diff --git a/acceptance/bundle/help/bundle-summary/output.txt b/acceptance/bundle/help/bundle-summary/output.txt
index 935c4bdc5..534bb8214 100644
--- a/acceptance/bundle/help/bundle-summary/output.txt
+++ b/acceptance/bundle/help/bundle-summary/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle summary --help
+>>> [CLI] bundle summary --help
 Summarize resources deployed by this bundle
 
 Usage:
diff --git a/acceptance/bundle/help/bundle-sync/output.txt b/acceptance/bundle/help/bundle-sync/output.txt
index 6588e6978..992138a20 100644
--- a/acceptance/bundle/help/bundle-sync/output.txt
+++ b/acceptance/bundle/help/bundle-sync/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle sync --help
+>>> [CLI] bundle sync --help
 Synchronize bundle tree to the workspace
 
 Usage:
diff --git a/acceptance/bundle/help/bundle-validate/output.txt b/acceptance/bundle/help/bundle-validate/output.txt
index a0c350faf..7fd1ae7ea 100644
--- a/acceptance/bundle/help/bundle-validate/output.txt
+++ b/acceptance/bundle/help/bundle-validate/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle validate --help
+>>> [CLI] bundle validate --help
 Validate configuration
 
 Usage:
diff --git a/acceptance/bundle/help/bundle/output.txt b/acceptance/bundle/help/bundle/output.txt
index e0e2ea47c..fc6dd623d 100644
--- a/acceptance/bundle/help/bundle/output.txt
+++ b/acceptance/bundle/help/bundle/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle --help
+>>> [CLI] bundle --help
 Databricks Asset Bundles let you express data/AI/analytics projects as code.
 
 Online documentation: https://docs.databricks.com/en/dev-tools/bundles/index.html
diff --git a/acceptance/bundle/override/clusters/output.txt b/acceptance/bundle/override/clusters/output.txt
index cff30b3af..a30a7bbff 100644
--- a/acceptance/bundle/override/clusters/output.txt
+++ b/acceptance/bundle/override/clusters/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle validate -o json -t default
+>>> [CLI] bundle validate -o json -t default
 {
   "autoscale": {
     "max_workers": 7,
@@ -15,7 +15,7 @@
   "spark_version": "13.3.x-scala2.12"
 }
 
->>> $CLI bundle validate -o json -t development
+>>> [CLI] bundle validate -o json -t development
 {
   "autoscale": {
     "max_workers": 3,
diff --git a/acceptance/bundle/override/job_cluster/output.txt b/acceptance/bundle/override/job_cluster/output.txt
index ff6e8316e..e4120e1c3 100644
--- a/acceptance/bundle/override/job_cluster/output.txt
+++ b/acceptance/bundle/override/job_cluster/output.txt
@@ -1,10 +1,10 @@
 
->>> $CLI bundle validate -o json -t development
+>>> [CLI] bundle validate -o json -t development
 {
   "foo": {
     "deployment": {
       "kind": "BUNDLE",
-      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/development/state/metadata.json"
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/override_job_cluster/development/state/metadata.json"
     },
     "edit_mode": "UI_LOCKED",
     "format": "MULTI_TASK",
@@ -27,12 +27,12 @@
   }
 }
 
->>> $CLI bundle validate -o json -t staging
+>>> [CLI] bundle validate -o json -t staging
 {
   "foo": {
     "deployment": {
       "kind": "BUNDLE",
-      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/staging/state/metadata.json"
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/override_job_cluster/staging/state/metadata.json"
     },
     "edit_mode": "UI_LOCKED",
     "format": "MULTI_TASK",
diff --git a/acceptance/bundle/override/job_cluster_var/output.txt b/acceptance/bundle/override/job_cluster_var/output.txt
index 0b19e5eb2..3545d6987 100644
--- a/acceptance/bundle/override/job_cluster_var/output.txt
+++ b/acceptance/bundle/override/job_cluster_var/output.txt
@@ -1,10 +1,10 @@
 
->>> $CLI bundle validate -o json -t development
+>>> [CLI] bundle validate -o json -t development
 {
   "foo": {
     "deployment": {
       "kind": "BUNDLE",
-      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/development/state/metadata.json"
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/override_job_cluster/development/state/metadata.json"
     },
     "edit_mode": "UI_LOCKED",
     "format": "MULTI_TASK",
@@ -27,21 +27,21 @@
   }
 }
 
->>> $CLI bundle validate -t development
+>>> [CLI] bundle validate -t development
 Name: override_job_cluster
 Target: development
 Workspace:
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/override_job_cluster/development
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/override_job_cluster/development
 
 Validation OK!
 
->>> $CLI bundle validate -o json -t staging
+>>> [CLI] bundle validate -o json -t staging
 {
   "foo": {
     "deployment": {
       "kind": "BUNDLE",
-      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/staging/state/metadata.json"
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/override_job_cluster/staging/state/metadata.json"
     },
     "edit_mode": "UI_LOCKED",
     "format": "MULTI_TASK",
@@ -64,11 +64,11 @@ Validation OK!
   }
 }
 
->>> $CLI bundle validate -t staging
+>>> [CLI] bundle validate -t staging
 Name: override_job_cluster
 Target: staging
 Workspace:
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/override_job_cluster/staging
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/override_job_cluster/staging
 
 Validation OK!
diff --git a/acceptance/bundle/override/job_tasks/out.development.stderr.txt b/acceptance/bundle/override/job_tasks/out.development.stderr.txt
index 7b6fef0cc..1873feb35 100644
--- a/acceptance/bundle/override/job_tasks/out.development.stderr.txt
+++ b/acceptance/bundle/override/job_tasks/out.development.stderr.txt
@@ -1,5 +1,5 @@
 
->>> errcode $CLI bundle validate -o json -t development
+>>> errcode [CLI] bundle validate -o json -t development
 Error: file ./test1.py not found
 
 
diff --git a/acceptance/bundle/override/job_tasks/output.txt b/acceptance/bundle/override/job_tasks/output.txt
index 915351d4e..1f7796217 100644
--- a/acceptance/bundle/override/job_tasks/output.txt
+++ b/acceptance/bundle/override/job_tasks/output.txt
@@ -28,7 +28,7 @@
   ]
 }
 
->>> errcode $CLI bundle validate -o json -t staging
+>>> errcode [CLI] bundle validate -o json -t staging
 Error: file ./test1.py not found
 
 
@@ -63,14 +63,14 @@ Exit code: 1
   ]
 }
 
->>> errcode $CLI bundle validate -t staging
+>>> errcode [CLI] bundle validate -t staging
 Error: file ./test1.py not found
 
 Name: override_job_tasks
 Target: staging
 Workspace:
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/override_job_tasks/staging
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/override_job_tasks/staging
 
 Found 1 error
 
diff --git a/acceptance/bundle/override/merge-string-map/output.txt b/acceptance/bundle/override/merge-string-map/output.txt
index b566aa07f..6e2aef87b 100644
--- a/acceptance/bundle/override/merge-string-map/output.txt
+++ b/acceptance/bundle/override/merge-string-map/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle validate -o json -t dev
+>>> [CLI] bundle validate -o json -t dev
 Warning: expected map, found string
   at resources.clusters.my_cluster
   in databricks.yml:6:17
@@ -13,7 +13,7 @@ Warning: expected map, found string
   }
 }
 
->>> $CLI bundle validate -t dev
+>>> [CLI] bundle validate -t dev
 Warning: expected map, found string
   at resources.clusters.my_cluster
   in databricks.yml:6:17
@@ -21,7 +21,7 @@ Warning: expected map, found string
 Name: merge-string-map
 Target: dev
 Workspace:
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/merge-string-map/dev
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/merge-string-map/dev
 
 Found 1 warning
diff --git a/acceptance/bundle/override/pipeline_cluster/output.txt b/acceptance/bundle/override/pipeline_cluster/output.txt
index 8babed0ec..d1a67f6b9 100644
--- a/acceptance/bundle/override/pipeline_cluster/output.txt
+++ b/acceptance/bundle/override/pipeline_cluster/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle validate -o json -t development
+>>> [CLI] bundle validate -o json -t development
 {
   "foo": {
     "clusters": [
@@ -14,14 +14,14 @@
     ],
     "deployment": {
       "kind": "BUNDLE",
-      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_pipeline_cluster/development/state/metadata.json"
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/override_pipeline_cluster/development/state/metadata.json"
     },
     "name": "job",
     "permissions": []
   }
 }
 
->>> $CLI bundle validate -o json -t staging
+>>> [CLI] bundle validate -o json -t staging
 {
   "foo": {
     "clusters": [
@@ -36,7 +36,7 @@
     ],
     "deployment": {
       "kind": "BUNDLE",
-      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_pipeline_cluster/staging/state/metadata.json"
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/override_pipeline_cluster/staging/state/metadata.json"
     },
     "name": "job",
     "permissions": []
diff --git a/acceptance/bundle/paths/fallback/output.job.json b/acceptance/bundle/paths/fallback/output.job.json
index fe9e1cf3d..ac79e0cf6 100644
--- a/acceptance/bundle/paths/fallback/output.job.json
+++ b/acceptance/bundle/paths/fallback/output.job.json
@@ -2,14 +2,14 @@
   {
     "job_cluster_key": "default",
     "notebook_task": {
-      "notebook_path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/notebook"
+      "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/notebook"
     },
     "task_key": "notebook_example"
   },
   {
     "job_cluster_key": "default",
     "spark_python_task": {
-      "python_file": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/file.py"
+      "python_file": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/file.py"
     },
     "task_key": "spark_python_example"
   },
@@ -19,7 +19,7 @@
         "dbt run",
         "dbt run"
       ],
-      "project_directory": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/dbt_project"
+      "project_directory": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/dbt_project"
     },
     "job_cluster_key": "default",
     "task_key": "dbt_example"
@@ -28,7 +28,7 @@
     "job_cluster_key": "default",
     "sql_task": {
       "file": {
-        "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/sql.sql"
+        "path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/sql.sql"
       },
       "warehouse_id": "cafef00d"
     },
diff --git a/acceptance/bundle/paths/fallback/output.pipeline.json b/acceptance/bundle/paths/fallback/output.pipeline.json
index 38521cb22..7ed4f74e6 100644
--- a/acceptance/bundle/paths/fallback/output.pipeline.json
+++ b/acceptance/bundle/paths/fallback/output.pipeline.json
@@ -1,22 +1,22 @@
 [
   {
     "file": {
-      "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/file1.py"
+      "path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/file1.py"
     }
   },
   {
     "notebook": {
-      "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/notebook1"
+      "path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/notebook1"
     }
   },
   {
     "file": {
-      "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/file2.py"
+      "path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/file2.py"
     }
   },
   {
     "notebook": {
-      "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/notebook2"
+      "path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/notebook2"
     }
   }
 ]
diff --git a/acceptance/bundle/paths/fallback/output.txt b/acceptance/bundle/paths/fallback/output.txt
index 63121f3d7..85f185851 100644
--- a/acceptance/bundle/paths/fallback/output.txt
+++ b/acceptance/bundle/paths/fallback/output.txt
@@ -1,15 +1,15 @@
 
->>> $CLI bundle validate -t development -o json
+>>> [CLI] bundle validate -t development -o json
 
->>> $CLI bundle validate -t error
+>>> [CLI] bundle validate -t error
 Error: notebook this value is overridden not found. Local notebook references are expected
 to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb]
 
 Name: fallback
 Target: error
 Workspace:
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/fallback/error
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/fallback/error
 
 Found 1 error
 
diff --git a/acceptance/bundle/paths/nominal/output.job.json b/acceptance/bundle/paths/nominal/output.job.json
index 9e1cb4d90..26d19d77c 100644
--- a/acceptance/bundle/paths/nominal/output.job.json
+++ b/acceptance/bundle/paths/nominal/output.job.json
@@ -2,14 +2,14 @@
   {
     "job_cluster_key": "default",
     "notebook_task": {
-      "notebook_path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook"
+      "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/notebook"
     },
     "task_key": "notebook_example"
   },
   {
     "job_cluster_key": "default",
     "spark_python_task": {
-      "python_file": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file.py"
+      "python_file": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/file.py"
     },
     "task_key": "spark_python_example"
   },
@@ -19,7 +19,7 @@
         "dbt run",
         "dbt run"
       ],
-      "project_directory": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/dbt_project"
+      "project_directory": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/dbt_project"
     },
     "job_cluster_key": "default",
     "task_key": "dbt_example"
@@ -28,7 +28,7 @@
     "job_cluster_key": "default",
     "sql_task": {
       "file": {
-        "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/sql.sql"
+        "path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/sql.sql"
       },
       "warehouse_id": "cafef00d"
     },
@@ -68,7 +68,7 @@
     "for_each_task": {
       "task": {
         "notebook_task": {
-          "notebook_path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook"
+          "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/notebook"
         }
       }
     },
@@ -80,7 +80,7 @@
       "task": {
         "job_cluster_key": "default",
         "spark_python_task": {
-          "python_file": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file.py"
+          "python_file": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/file.py"
         }
       }
     },
diff --git a/acceptance/bundle/paths/nominal/output.pipeline.json b/acceptance/bundle/paths/nominal/output.pipeline.json
index 277b0c4a1..c6f2e0868 100644
--- a/acceptance/bundle/paths/nominal/output.pipeline.json
+++ b/acceptance/bundle/paths/nominal/output.pipeline.json
@@ -1,22 +1,22 @@
 [
   {
     "file": {
-      "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file1.py"
+      "path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/file1.py"
     }
   },
   {
     "notebook": {
-      "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook1"
+      "path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/notebook1"
     }
   },
   {
     "file": {
-      "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file2.py"
+      "path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/file2.py"
     }
   },
   {
     "notebook": {
-      "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook2"
+      "path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/notebook2"
     }
   }
 ]
diff --git a/acceptance/bundle/paths/nominal/output.txt b/acceptance/bundle/paths/nominal/output.txt
index 1badcdec6..40670f4cb 100644
--- a/acceptance/bundle/paths/nominal/output.txt
+++ b/acceptance/bundle/paths/nominal/output.txt
@@ -1,15 +1,15 @@
 
->>> $CLI bundle validate -t development -o json
+>>> [CLI] bundle validate -t development -o json
 
->>> $CLI bundle validate -t error
+>>> [CLI] bundle validate -t error
 Error: notebook this value is overridden not found. Local notebook references are expected
 to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb]
 
 Name: nominal
 Target: error
 Workspace:
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/nominal/error
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/nominal/error
 
 Found 1 error
 
diff --git a/acceptance/bundle/paths/relative_path_translation/output.txt b/acceptance/bundle/paths/relative_path_translation/output.txt
index 362f2ec7b..b13d612b6 100644
--- a/acceptance/bundle/paths/relative_path_translation/output.txt
+++ b/acceptance/bundle/paths/relative_path_translation/output.txt
@@ -1,4 +1,4 @@
 
->>> $CLI bundle validate -t default -o json
+>>> [CLI] bundle validate -t default -o json
 
->>> $CLI bundle validate -t override -o json
+>>> [CLI] bundle validate -t override -o json
diff --git a/acceptance/bundle/quality_monitor/output.txt b/acceptance/bundle/quality_monitor/output.txt
index b3718c802..8a7f64ef2 100644
--- a/acceptance/bundle/quality_monitor/output.txt
+++ b/acceptance/bundle/quality_monitor/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle validate -o json -t development
+>>> [CLI] bundle validate -o json -t development
 {
   "mode": "development",
   "quality_monitors": {
@@ -21,7 +21,7 @@
   }
 }
 
->>> $CLI bundle validate -o json -t staging
+>>> [CLI] bundle validate -o json -t staging
 {
   "mode": null,
   "quality_monitors": {
@@ -46,7 +46,7 @@
   }
 }
 
->>> $CLI bundle validate -o json -t production
+>>> [CLI] bundle validate -o json -t production
 {
   "mode": null,
   "quality_monitors": {
diff --git a/acceptance/bundle/scripts/output.txt b/acceptance/bundle/scripts/output.txt
index ec5978380..2deedb0e7 100644
--- a/acceptance/bundle/scripts/output.txt
+++ b/acceptance/bundle/scripts/output.txt
@@ -1,5 +1,5 @@
 
->>> EXITCODE=0 errcode $CLI bundle validate
+>>> EXITCODE=0 errcode [CLI] bundle validate
 Executing 'preinit' script
 from myscript.py 0 preinit: hello stdout!
 from myscript.py 0 preinit: hello stderr!
@@ -9,12 +9,12 @@ from myscript.py 0 postinit: hello stderr!
 Name: scripts
 Target: default
 Workspace:
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/scripts/default
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/scripts/default
 
 Validation OK!
 
->>> EXITCODE=1 errcode $CLI bundle validate
+>>> EXITCODE=1 errcode [CLI] bundle validate
 Executing 'preinit' script
 from myscript.py 1 preinit: hello stdout!
 from myscript.py 1 preinit: hello stderr!
@@ -26,7 +26,7 @@ Found 1 error
 
 Exit code: 1
 
->>> EXITCODE=0 errcode $CLI bundle deploy
+>>> EXITCODE=0 errcode [CLI] bundle deploy
 Executing 'preinit' script
 from myscript.py 0 preinit: hello stdout!
 from myscript.py 0 preinit: hello stderr!
@@ -42,7 +42,7 @@ from myscript.py 0 postbuild: hello stderr!
 Executing 'predeploy' script
 from myscript.py 0 predeploy: hello stdout!
 from myscript.py 0 predeploy: hello stderr!
-Error: unable to deploy to /Workspace/Users/$USERNAME/.bundle/scripts/default/state as $USERNAME.
+Error: unable to deploy to /Workspace/Users/[USERNAME]/.bundle/scripts/default/state as [USERNAME].
 Please make sure the current user or one of their groups is listed under the permissions of this bundle.
 For assistance, contact the owners of this project.
 They may need to redeploy the bundle to apply the new permissions.
diff --git a/acceptance/bundle/syncroot/dotdot-git/output.txt b/acceptance/bundle/syncroot/dotdot-git/output.txt
index f1dc5fb01..dbfc8451f 100644
--- a/acceptance/bundle/syncroot/dotdot-git/output.txt
+++ b/acceptance/bundle/syncroot/dotdot-git/output.txt
@@ -1,10 +1,10 @@
-Error: path "$TMPDIR" is not within repository root "$TMPDIR/myrepo"
+Error: path "[TMPDIR]" is not within repository root "[TMPDIR]/myrepo"
 
 Name: test-bundle
 Target: default
 Workspace:
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/test-bundle/default
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
 
 Found 1 error
 
diff --git a/acceptance/bundle/syncroot/dotdot-nogit/output.txt b/acceptance/bundle/syncroot/dotdot-nogit/output.txt
index 46f617f35..4f189effd 100644
--- a/acceptance/bundle/syncroot/dotdot-nogit/output.txt
+++ b/acceptance/bundle/syncroot/dotdot-nogit/output.txt
@@ -1,7 +1,7 @@
 Name: test-bundle
 Target: default
 Workspace:
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/test-bundle/default
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
 
 Validation OK!
diff --git a/acceptance/bundle/templates/dbt-sql/output.txt b/acceptance/bundle/templates/dbt-sql/output.txt
index 972c7e152..2699ad554 100644
--- a/acceptance/bundle/templates/dbt-sql/output.txt
+++ b/acceptance/bundle/templates/dbt-sql/output.txt
@@ -1,32 +1,32 @@
 
->>> $CLI bundle init dbt-sql --config-file ./input.json --output-dir output
+>>> [CLI] bundle init dbt-sql --config-file ./input.json --output-dir output
 
 Welcome to the dbt template for Databricks Asset Bundles!
 
 A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html.
-workspace_host: $DATABRICKS_URL
+workspace_host: [DATABRICKS_URL]
 
 📊 Your new project has been created in the 'my_dbt_sql' directory!
 If you already have dbt installed, just type 'cd my_dbt_sql; dbt init' to get started.
 Refer to the README.md file for full "getting started" guide and production setup instructions.
 
 
->>> $CLI bundle validate -t dev
+>>> [CLI] bundle validate -t dev
 Name: my_dbt_sql
 Target: dev
 Workspace:
-  Host: $DATABRICKS_URL
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/my_dbt_sql/dev
+  Host: [DATABRICKS_URL]
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/my_dbt_sql/dev
 
 Validation OK!
 
->>> $CLI bundle validate -t prod
+>>> [CLI] bundle validate -t prod
 Name: my_dbt_sql
 Target: prod
 Workspace:
-  Host: $DATABRICKS_URL
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/my_dbt_sql/prod
+  Host: [DATABRICKS_URL]
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/my_dbt_sql/prod
 
 Validation OK!
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml
index cdf3704b9..3651ef12d 100644
--- a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml
+++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml
@@ -19,16 +19,16 @@ targets:
     # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html.
     mode: development
     workspace:
-      host: $DATABRICKS_URL
+      host: [DATABRICKS_URL]
 
   prod:
     mode: production
     workspace:
-      host: $DATABRICKS_URL
-      # We explicitly specify /Workspace/Users/$USERNAME to make sure we only have a single copy.
-      root_path: /Workspace/Users/$USERNAME/.bundle/${bundle.name}/${bundle.target}
+      host: [DATABRICKS_URL]
+      # We explicitly specify /Workspace/Users/[USERNAME] to make sure we only have a single copy.
+      root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target}
     permissions:
-      - user_name: $USERNAME
+      - user_name: [USERNAME]
         level: CAN_MANAGE
     run_as:
-      user_name: $USERNAME
+      user_name: [USERNAME]
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/profile_template.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/profile_template.yml
index 5e0f0fc29..bdb41ab20 100644
--- a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/profile_template.yml
+++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/profile_template.yml
@@ -5,7 +5,7 @@ fixed:
   type: databricks
 prompts:
   host:
-    default: $DATABRICKS_HOST
+    default: [DATABRICKS_HOST]
   token:
     hint: 'personal access token to use, dapiXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
     hide_input: true
@@ -16,7 +16,7 @@ prompts:
     hint: 'initial catalog'
     default: main
   schema:
-    hint: 'personal schema where dbt will build objects during development, example: $USERNAME'
+    hint: 'personal schema where dbt will build objects during development, example: [USERNAME]'
   threads:
     hint: 'threads to use during development, 1 or more'
     type: 'int'
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml
index d52f8ed50..b522931f9 100644
--- a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml
+++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml
@@ -11,7 +11,7 @@ resources:
 
       email_notifications:
         on_failure:
-          - $USERNAME
+          - [USERNAME]
 
 
       tasks:
diff --git a/acceptance/bundle/templates/default-python/output.txt b/acceptance/bundle/templates/default-python/output.txt
index 5493ac2cf..930e756de 100644
--- a/acceptance/bundle/templates/default-python/output.txt
+++ b/acceptance/bundle/templates/default-python/output.txt
@@ -1,30 +1,30 @@
 
->>> $CLI bundle init default-python --config-file ./input.json --output-dir output
+>>> [CLI] bundle init default-python --config-file ./input.json --output-dir output
 
 Welcome to the default Python template for Databricks Asset Bundles!
-Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): $DATABRICKS_URL
+Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): [DATABRICKS_URL]
 
 ✨ Your new project has been created in the 'my_default_python' directory!
 
 Please refer to the README.md file for "getting started" instructions.
 See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.
 
->>> $CLI bundle validate -t dev
+>>> [CLI] bundle validate -t dev
 Name: my_default_python
 Target: dev
 Workspace:
-  Host: $DATABRICKS_URL
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/my_default_python/dev
+  Host: [DATABRICKS_URL]
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev
 
 Validation OK!
 
->>> $CLI bundle validate -t prod
+>>> [CLI] bundle validate -t prod
 Name: my_default_python
 Target: prod
 Workspace:
-  Host: $DATABRICKS_URL
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/my_default_python/prod
+  Host: [DATABRICKS_URL]
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod
 
 Validation OK!
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml b/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml
index 3fa777219..6df75c209 100644
--- a/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml
@@ -16,16 +16,16 @@ targets:
     mode: development
     default: true
     workspace:
-      host: $DATABRICKS_URL
+      host: [DATABRICKS_URL]
 
   prod:
     mode: production
     workspace:
-      host: $DATABRICKS_URL
-      # We explicitly specify /Workspace/Users/$USERNAME to make sure we only have a single copy.
-      root_path: /Workspace/Users/$USERNAME/.bundle/${bundle.name}/${bundle.target}
+      host: [DATABRICKS_URL]
+      # We explicitly specify /Workspace/Users/[USERNAME] to make sure we only have a single copy.
+      root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target}
     permissions:
-      - user_name: $USERNAME
+      - user_name: [USERNAME]
         level: CAN_MANAGE
     run_as:
-      user_name: $USERNAME
+      user_name: [USERNAME]
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml b/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml
index e6148a4ad..d9e31691a 100644
--- a/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml
@@ -12,7 +12,7 @@ resources:
 
       email_notifications:
         on_failure:
-          - $USERNAME
+          - [USERNAME]
 
       tasks:
         - task_key: notebook_task
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/setup.py b/acceptance/bundle/templates/default-python/output/my_default_python/setup.py
index 84b24ecb8..548f1035e 100644
--- a/acceptance/bundle/templates/default-python/output/my_default_python/setup.py
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/setup.py
@@ -23,7 +23,7 @@ setup(
     # to ensure that changes to wheel package are picked up when used on all-purpose clusters
     version=my_default_python.__version__ + "+" + local_version,
     url="https://databricks.com",
-    author="$USERNAME",
+    author="[USERNAME]",
     description="wheel file based on my_default_python/src",
     packages=find_packages(where="./src"),
     package_dir={"": "src"},
diff --git a/acceptance/bundle/templates/default-sql/output.txt b/acceptance/bundle/templates/default-sql/output.txt
index fe0139093..06eff962b 100644
--- a/acceptance/bundle/templates/default-sql/output.txt
+++ b/acceptance/bundle/templates/default-sql/output.txt
@@ -1,32 +1,32 @@
 
->>> $CLI bundle init default-sql --config-file ./input.json --output-dir output
+>>> [CLI] bundle init default-sql --config-file ./input.json --output-dir output
 
 Welcome to the default SQL template for Databricks Asset Bundles!
 
 A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html.
-workspace_host: $DATABRICKS_URL
+workspace_host: [DATABRICKS_URL]
 
 ✨ Your new project has been created in the 'my_default_sql' directory!
 
 Please refer to the README.md file for "getting started" instructions.
 See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.
 
->>> $CLI bundle validate -t dev
+>>> [CLI] bundle validate -t dev
 Name: my_default_sql
 Target: dev
 Workspace:
-  Host: $DATABRICKS_URL
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/my_default_sql/dev
+  Host: [DATABRICKS_URL]
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/my_default_sql/dev
 
 Validation OK!
 
->>> $CLI bundle validate -t prod
+>>> [CLI] bundle validate -t prod
 Name: my_default_sql
 Target: prod
 Workspace:
-  Host: $DATABRICKS_URL
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/my_default_sql/prod
+  Host: [DATABRICKS_URL]
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/my_default_sql/prod
 
 Validation OK!
diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml b/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml
index 16292bc84..6ef09cf3b 100644
--- a/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml
+++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml
@@ -25,7 +25,7 @@ targets:
     mode: development
     default: true
     workspace:
-      host: $DATABRICKS_URL
+      host: [DATABRICKS_URL]
     variables:
       warehouse_id: f00dcafe
       catalog: main
@@ -34,15 +34,15 @@ targets:
   prod:
     mode: production
     workspace:
-      host: $DATABRICKS_URL
-      # We explicitly specify /Workspace/Users/$USERNAME to make sure we only have a single copy.
-      root_path: /Workspace/Users/$USERNAME/.bundle/${bundle.name}/${bundle.target}
+      host: [DATABRICKS_URL]
+      # We explicitly specify /Workspace/Users/[USERNAME] to make sure we only have a single copy.
+      root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target}
     variables:
       warehouse_id: f00dcafe
       catalog: main
       schema: default
     permissions:
-      - user_name: $USERNAME
+      - user_name: [USERNAME]
         level: CAN_MANAGE
     run_as:
-      user_name: $USERNAME
+      user_name: [USERNAME]
diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/resources/my_default_sql_sql.job.yml b/acceptance/bundle/templates/default-sql/output/my_default_sql/resources/my_default_sql_sql.job.yml
index 86de0f9db..34d60e3d5 100644
--- a/acceptance/bundle/templates/default-sql/output/my_default_sql/resources/my_default_sql_sql.job.yml
+++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/resources/my_default_sql_sql.job.yml
@@ -12,7 +12,7 @@ resources:
 
       email_notifications:
         on_failure:
-          - $USERNAME
+          - [USERNAME]
 
       parameters:
         - name: catalog
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output.txt b/acceptance/bundle/templates/experimental-jobs-as-code/output.txt
index 10aca003e..984dad604 100644
--- a/acceptance/bundle/templates/experimental-jobs-as-code/output.txt
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output.txt
@@ -1,28 +1,28 @@
 
->>> $CLI bundle init experimental-jobs-as-code --config-file ./input.json --output-dir output
+>>> [CLI] bundle init experimental-jobs-as-code --config-file ./input.json --output-dir output
 
 Welcome to (EXPERIMENTAL) "Jobs as code" template for Databricks Asset Bundles!
-Workspace to use (auto-detected, edit in 'my_jobs_as_code/databricks.yml'): $DATABRICKS_URL
+Workspace to use (auto-detected, edit in 'my_jobs_as_code/databricks.yml'): [DATABRICKS_URL]
 
 ✨ Your new project has been created in the 'my_jobs_as_code' directory!
 
 Please refer to the README.md file for "getting started" instructions.
 See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.
 
->>> $CLI bundle validate -t dev --output json
-Warning: Ignoring Databricks CLI version constraint for development build. Required: >= 0.238.0, current: $DEV_VERSION
+>>> [CLI] bundle validate -t dev --output json
+Warning: Ignoring Databricks CLI version constraint for development build. Required: >= 0.238.0, current: [DEV_VERSION]
 
 {
   "jobs": {
     "my_jobs_as_code_job": {
       "deployment": {
         "kind": "BUNDLE",
-        "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/my_jobs_as_code/dev/state/metadata.json"
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_jobs_as_code/dev/state/metadata.json"
       },
       "edit_mode": "UI_LOCKED",
       "email_notifications": {
         "on_failure": [
-          "$USERNAME"
+          "[USERNAME]"
         ]
       },
       "format": "MULTI_TASK",
@@ -40,19 +40,19 @@ Warning: Ignoring Databricks CLI version constraint for development build. Requi
         }
       ],
       "max_concurrent_runs": 4,
-      "name": "[dev $USERNAME] my_jobs_as_code_job",
+      "name": "[dev [USERNAME]] my_jobs_as_code_job",
       "permissions": [],
       "queue": {
         "enabled": true
       },
       "tags": {
-        "dev": "$USERNAME"
+        "dev": "[USERNAME]"
       },
       "tasks": [
         {
           "job_cluster_key": "job_cluster",
           "notebook_task": {
-            "notebook_path": "/Workspace/Users/$USERNAME/.bundle/my_jobs_as_code/dev/files/src/notebook"
+            "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_jobs_as_code/dev/files/src/notebook"
           },
           "task_key": "notebook_task"
         },
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml
index 54e69a256..9299c96e8 100644
--- a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml
@@ -34,16 +34,16 @@ targets:
     mode: development
     default: true
     workspace:
-      host: $DATABRICKS_URL
+      host: [DATABRICKS_URL]
 
   prod:
     mode: production
     workspace:
-      host: $DATABRICKS_URL
-      # We explicitly specify /Workspace/Users/$USERNAME to make sure we only have a single copy.
-      root_path: /Workspace/Users/$USERNAME/.bundle/${bundle.name}/${bundle.target}
+      host: [DATABRICKS_URL]
+      # We explicitly specify /Workspace/Users/[USERNAME] to make sure we only have a single copy.
+      root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target}
     permissions:
-      - user_name: $USERNAME
+      - user_name: [USERNAME]
         level: CAN_MANAGE
     run_as:
-      user_name: $USERNAME
+      user_name: [USERNAME]
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py
index 4854d656f..e8406fd7b 100644
--- a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py
@@ -17,7 +17,7 @@ my_jobs_as_code_job = Job.from_dict(
         },
         "email_notifications": {
             "on_failure": [
-                "$USERNAME",
+                "[USERNAME]",
             ],
         },
         "tasks": [
diff --git a/acceptance/bundle/templates/wrong-url/output.txt b/acceptance/bundle/templates/wrong-url/output.txt
index b78cf4b68..6b4f9c459 100644
--- a/acceptance/bundle/templates/wrong-url/output.txt
+++ b/acceptance/bundle/templates/wrong-url/output.txt
@@ -1,4 +1,4 @@
-Error: git clone failed: git clone https://invalid-domain-123.databricks.com/hello/world $TMPDIR_GPARENT/world-123456 --no-tags --depth=1: exit status 128. Cloning into '$TMPDIR_GPARENT/world-123456'...
+Error: git clone failed: git clone https://invalid-domain-123.databricks.com/hello/world [TMPDIR]_GPARENT/world-123456 --no-tags --depth=1: exit status 128. Cloning into '[TMPDIR]_GPARENT/world-123456'...
 fatal: unable to access 'https://invalid-domain-123.databricks.com/hello/world/': Could not resolve host: invalid-domain-123.databricks.com
 
 
diff --git a/acceptance/bundle/variables/arg-repeat/output.txt b/acceptance/bundle/variables/arg-repeat/output.txt
index 2f9de1a3c..4b97d70a1 100644
--- a/acceptance/bundle/variables/arg-repeat/output.txt
+++ b/acceptance/bundle/variables/arg-repeat/output.txt
@@ -1,5 +1,5 @@
 
->>> errcode $CLI bundle validate --var a=one -o json
+>>> errcode [CLI] bundle validate --var a=one -o json
 {
   "a": {
     "default": "hello",
@@ -7,7 +7,7 @@
   }
 }
 
->>> errcode $CLI bundle validate --var a=one --var a=two
+>>> errcode [CLI] bundle validate --var a=one --var a=two
 Error: failed to assign two to a: variable has already been assigned value: one
 
 Name: arg-repeat
diff --git a/acceptance/bundle/variables/complex-cycle-self/output.txt b/acceptance/bundle/variables/complex-cycle-self/output.txt
index fa80154ca..7447de349 100644
--- a/acceptance/bundle/variables/complex-cycle-self/output.txt
+++ b/acceptance/bundle/variables/complex-cycle-self/output.txt
@@ -3,7 +3,7 @@ Warning: Detected unresolved variables after 11 resolution rounds
 Name: cycle
 Target: default
 Workspace:
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/cycle/default
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/cycle/default
 
 Found 1 warning
diff --git a/acceptance/bundle/variables/complex-cycle/output.txt b/acceptance/bundle/variables/complex-cycle/output.txt
index fa80154ca..7447de349 100644
--- a/acceptance/bundle/variables/complex-cycle/output.txt
+++ b/acceptance/bundle/variables/complex-cycle/output.txt
@@ -3,7 +3,7 @@ Warning: Detected unresolved variables after 11 resolution rounds
 Name: cycle
 Target: default
 Workspace:
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/cycle/default
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/cycle/default
 
 Found 1 warning
diff --git a/acceptance/bundle/variables/complex/out.default.json b/acceptance/bundle/variables/complex/out.default.json
index a1ccd52bc..0804ad588 100644
--- a/acceptance/bundle/variables/complex/out.default.json
+++ b/acceptance/bundle/variables/complex/out.default.json
@@ -4,7 +4,7 @@
       "my_job": {
         "deployment": {
           "kind": "BUNDLE",
-          "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/complex-variables/default/state/metadata.json"
+          "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/complex-variables/default/state/metadata.json"
         },
         "edit_mode": "UI_LOCKED",
         "format": "MULTI_TASK",
diff --git a/acceptance/bundle/variables/complex/out.dev.json b/acceptance/bundle/variables/complex/out.dev.json
index bb939091b..e93c2c297 100644
--- a/acceptance/bundle/variables/complex/out.dev.json
+++ b/acceptance/bundle/variables/complex/out.dev.json
@@ -4,7 +4,7 @@
       "my_job": {
         "deployment": {
           "kind": "BUNDLE",
-          "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/complex-variables/dev/state/metadata.json"
+          "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/complex-variables/dev/state/metadata.json"
         },
         "edit_mode": "UI_LOCKED",
         "format": "MULTI_TASK",
diff --git a/acceptance/bundle/variables/complex/output.txt b/acceptance/bundle/variables/complex/output.txt
index ce295421f..f1d4c04cc 100644
--- a/acceptance/bundle/variables/complex/output.txt
+++ b/acceptance/bundle/variables/complex/output.txt
@@ -1,10 +1,10 @@
 
->>> $CLI bundle validate -o json
+>>> [CLI] bundle validate -o json
 
 >>> jq .resources.jobs.my_job.tasks[0].task_key out.default.json
 "task with spark version 13.2.x-scala2.11 and jar /path/to/jar"
 
->>> $CLI bundle validate -o json -t dev
+>>> [CLI] bundle validate -o json -t dev
 
 >>> jq .resources.jobs.my_job.tasks[0].task_key out.dev.json
 "task with spark version 14.2.x-scala2.11 and jar /newpath/to/jar"
diff --git a/acceptance/bundle/variables/complex_multiple_files/output.txt b/acceptance/bundle/variables/complex_multiple_files/output.txt
index ec2cad1ce..433e6da0c 100644
--- a/acceptance/bundle/variables/complex_multiple_files/output.txt
+++ b/acceptance/bundle/variables/complex_multiple_files/output.txt
@@ -4,7 +4,7 @@
       "my_job": {
         "deployment": {
           "kind": "BUNDLE",
-          "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/complex-variables-multiple-files/dev/state/metadata.json"
+          "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/complex-variables-multiple-files/dev/state/metadata.json"
         },
         "edit_mode": "UI_LOCKED",
         "format": "MULTI_TASK",
diff --git a/acceptance/bundle/variables/double_underscore/output.txt b/acceptance/bundle/variables/double_underscore/output.txt
index 45529038d..0124f5442 100644
--- a/acceptance/bundle/variables/double_underscore/output.txt
+++ b/acceptance/bundle/variables/double_underscore/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle validate -o json
+>>> [CLI] bundle validate -o json
 [
   {
     "task_key": "test default"
diff --git a/acceptance/bundle/variables/empty/output.txt b/acceptance/bundle/variables/empty/output.txt
index 8933443df..cbd0f1989 100644
--- a/acceptance/bundle/variables/empty/output.txt
+++ b/acceptance/bundle/variables/empty/output.txt
@@ -3,8 +3,8 @@ Error: no value assigned to required variable a. Assignment can be done using "-
 Name: empty${var.a}
 Target: default
 Workspace:
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/empty${var.a}/default
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/empty${var.a}/default
 
 Found 1 error
 
diff --git a/acceptance/bundle/variables/env_overrides/output.txt b/acceptance/bundle/variables/env_overrides/output.txt
index 06e6e518b..93b3b6716 100644
--- a/acceptance/bundle/variables/env_overrides/output.txt
+++ b/acceptance/bundle/variables/env_overrides/output.txt
@@ -1,27 +1,27 @@
 
->>> $CLI bundle validate -t env-with-single-variable-override -o json
+>>> [CLI] bundle validate -t env-with-single-variable-override -o json
 "default-a dev-b"
 
->>> $CLI bundle validate -t env-with-two-variable-overrides -o json
+>>> [CLI] bundle validate -t env-with-two-variable-overrides -o json
 "prod-a prod-b"
 
->>> BUNDLE_VAR_b=env-var-b $CLI bundle validate -t env-with-two-variable-overrides -o json
+>>> BUNDLE_VAR_b=env-var-b [CLI] bundle validate -t env-with-two-variable-overrides -o json
 "prod-a env-var-b"
 
->>> errcode $CLI bundle validate -t env-missing-a-required-variable-assignment
+>>> errcode [CLI] bundle validate -t env-missing-a-required-variable-assignment
 Error: no value assigned to required variable b. Assignment can be done using "--var", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle/<target>/variable-overrides.json file
 
 Name: test bundle
 Target: env-missing-a-required-variable-assignment
 Workspace:
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/test bundle/env-missing-a-required-variable-assignment
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test bundle/env-missing-a-required-variable-assignment
 
 Found 1 error
 
 Exit code: 1
 
->>> errcode $CLI bundle validate -t env-using-an-undefined-variable
+>>> errcode [CLI] bundle validate -t env-using-an-undefined-variable
 Error: variable c is not defined but is assigned a value
 
 Name: test bundle
@@ -30,7 +30,7 @@ Found 1 error
 
 Exit code: 1
 
->>> $CLI bundle validate -t env-overrides-lookup -o json
+>>> [CLI] bundle validate -t env-overrides-lookup -o json
 {
   "a": "default-a",
   "b": "prod-b",
diff --git a/acceptance/bundle/variables/file-defaults/output.txt b/acceptance/bundle/variables/file-defaults/output.txt
index 5b01a1b66..234ddcbbd 100644
--- a/acceptance/bundle/variables/file-defaults/output.txt
+++ b/acceptance/bundle/variables/file-defaults/output.txt
@@ -1,6 +1,6 @@
 
 === variable file
->>> $CLI bundle validate -o json
+>>> [CLI] bundle validate -o json
 {
   "job_cluster_key": "mlops_stacks-cluster",
   "new_cluster": {
@@ -10,7 +10,7 @@
 }
 
 === variable file and variable flag
->>> $CLI bundle validate -o json --var=cluster_key=mlops_stacks-cluster-overriden
+>>> [CLI] bundle validate -o json --var=cluster_key=mlops_stacks-cluster-overriden
 {
   "job_cluster_key": "mlops_stacks-cluster-overriden",
   "new_cluster": {
@@ -20,7 +20,7 @@
 }
 
 === variable file and environment variable
->>> BUNDLE_VAR_cluster_key=mlops_stacks-cluster-overriden $CLI bundle validate -o json
+>>> BUNDLE_VAR_cluster_key=mlops_stacks-cluster-overriden [CLI] bundle validate -o json
 {
   "job_cluster_key": "mlops_stacks-cluster-overriden",
   "new_cluster": {
@@ -30,7 +30,7 @@
 }
 
 === variable has value in config file
->>> $CLI bundle validate -o json --target with_value
+>>> [CLI] bundle validate -o json --target with_value
 {
   "job_cluster_key": "mlops_stacks-cluster-from-file",
   "new_cluster": {
@@ -40,8 +40,8 @@
 }
 
 === file cannot be parsed
->>> errcode $CLI bundle validate -o json --target invalid_json
-Error: failed to parse variables file $TMPDIR/.databricks/bundle/invalid_json/variable-overrides.json: error decoding JSON at :0:0: invalid character 'o' in literal false (expecting 'a')
+>>> errcode [CLI] bundle validate -o json --target invalid_json
+Error: failed to parse variables file [TMPDIR]/.databricks/bundle/invalid_json/variable-overrides.json: error decoding JSON at :0:0: invalid character 'o' in literal false (expecting 'a')
 
 
 Exit code: 1
@@ -54,8 +54,8 @@ Exit code: 1
 }
 
 === file has wrong structure
->>> errcode $CLI bundle validate -o json --target wrong_file_structure
-Error: failed to parse variables file $TMPDIR/.databricks/bundle/wrong_file_structure/variable-overrides.json: invalid format
+>>> errcode [CLI] bundle validate -o json --target wrong_file_structure
+Error: failed to parse variables file [TMPDIR]/.databricks/bundle/wrong_file_structure/variable-overrides.json: invalid format
 
 Variables file must be a JSON object with the following format:
 {"var1": "value1", "var2": "value2"}
@@ -71,7 +71,7 @@ Exit code: 1
 }
 
 === file has variable that is complex but default is string
->>> errcode $CLI bundle validate -o json --target complex_to_string
+>>> errcode [CLI] bundle validate -o json --target complex_to_string
 Error: variable cluster_key is not of type complex, but the value in the variable file is a complex type
 
 
@@ -85,7 +85,7 @@ Exit code: 1
 }
 
 === file has variable that is string but default is complex
->>> errcode $CLI bundle validate -o json --target string_to_complex
+>>> errcode [CLI] bundle validate -o json --target string_to_complex
 Error: variable cluster is of type complex, but the value in the variable file is not a complex type
 
 
@@ -99,7 +99,7 @@ Exit code: 1
 }
 
 === variable is required but it's not provided in the file
->>> errcode $CLI bundle validate -o json --target without_defaults
+>>> errcode [CLI] bundle validate -o json --target without_defaults
 Error: no value assigned to required variable cluster. Assignment can be done using "--var", by setting the BUNDLE_VAR_cluster environment variable, or in .databricks/bundle/<target>/variable-overrides.json file
 
 
diff --git a/acceptance/bundle/variables/file-defaults/test.toml b/acceptance/bundle/variables/file-defaults/test.toml
index 08403b606..da8854775 100644
--- a/acceptance/bundle/variables/file-defaults/test.toml
+++ b/acceptance/bundle/variables/file-defaults/test.toml
@@ -1,8 +1,4 @@
 # Fix for windows
 [[Repls]]
-Old = '\$TMPDIR\\.databricks\\bundle\\wrong_file_structure\\variable-overrides.json'
-New = '$$TMPDIR/.databricks/bundle/wrong_file_structure/variable-overrides.json'
-
-[[Repls]]
-Old = '\$TMPDIR\\.databricks\\bundle\\invalid_json\\variable-overrides.json'
-New = '$$TMPDIR/.databricks/bundle/invalid_json/variable-overrides.json'
+Old = '\\'
+New = '/'
diff --git a/acceptance/bundle/variables/git-branch/output.txt b/acceptance/bundle/variables/git-branch/output.txt
index 21ed9e7de..68f27a3f1 100644
--- a/acceptance/bundle/variables/git-branch/output.txt
+++ b/acceptance/bundle/variables/git-branch/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle validate -o json
+>>> [CLI] bundle validate -o json
 {
   "bundle": {
     "environment": "prod",
@@ -11,7 +11,7 @@
     "name": "git",
     "target": "prod",
     "terraform": {
-      "exec_path": "$TERRAFORM"
+      "exec_path": "[TERRAFORM]"
     }
   },
   "sync": {
@@ -27,24 +27,24 @@
     }
   },
   "workspace": {
-    "artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/artifacts",
-    "file_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/files",
-    "resource_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/resources",
-    "root_path": "/Workspace/Users/$USERNAME/.bundle/git/prod",
-    "state_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/state"
+    "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/git/prod/artifacts",
+    "file_path": "/Workspace/Users/[USERNAME]/.bundle/git/prod/files",
+    "resource_path": "/Workspace/Users/[USERNAME]/.bundle/git/prod/resources",
+    "root_path": "/Workspace/Users/[USERNAME]/.bundle/git/prod",
+    "state_path": "/Workspace/Users/[USERNAME]/.bundle/git/prod/state"
   }
 }
 
->>> $CLI bundle validate
+>>> [CLI] bundle validate
 Name: git
 Target: prod
 Workspace:
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/git/prod
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/git/prod
 
 Validation OK!
 
->>> $CLI bundle validate -o json -t dev
+>>> [CLI] bundle validate -o json -t dev
 {
   "bundle": {
     "environment": "dev",
@@ -56,7 +56,7 @@ Validation OK!
     "name": "git",
     "target": "dev",
     "terraform": {
-      "exec_path": "$TERRAFORM"
+      "exec_path": "[TERRAFORM]"
     }
   },
   "sync": {
@@ -72,19 +72,19 @@ Validation OK!
     }
   },
   "workspace": {
-    "artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/artifacts",
-    "file_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/files",
-    "resource_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/resources",
-    "root_path": "/Workspace/Users/$USERNAME/.bundle/git/dev",
-    "state_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/state"
+    "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/git/dev/artifacts",
+    "file_path": "/Workspace/Users/[USERNAME]/.bundle/git/dev/files",
+    "resource_path": "/Workspace/Users/[USERNAME]/.bundle/git/dev/resources",
+    "root_path": "/Workspace/Users/[USERNAME]/.bundle/git/dev",
+    "state_path": "/Workspace/Users/[USERNAME]/.bundle/git/dev/state"
   }
 }
 
->>> $CLI bundle validate -t dev
+>>> [CLI] bundle validate -t dev
 Name: git
 Target: dev
 Workspace:
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/git/dev
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/git/dev
 
 Validation OK!
diff --git a/acceptance/bundle/variables/host/output.txt b/acceptance/bundle/variables/host/output.txt
index 89342908c..63c41426a 100644
--- a/acceptance/bundle/variables/host/output.txt
+++ b/acceptance/bundle/variables/host/output.txt
@@ -1,5 +1,5 @@
 
->>> errcode $CLI bundle validate -o json
+>>> errcode [CLI] bundle validate -o json
 Error: failed during request visitor: parse "https://${var.host}": invalid character "{" in host name
 
 {
@@ -25,7 +25,7 @@ Error: failed during request visitor: parse "https://${var.host}": invalid chara
 }
 Exit code: 1
 
->>> errcode $CLI bundle validate
+>>> errcode [CLI] bundle validate
 Error: failed during request visitor: parse "https://${var.host}": invalid character "{" in host name
 
 Name: host
diff --git a/acceptance/bundle/variables/prepend-workspace-var/output.txt b/acceptance/bundle/variables/prepend-workspace-var/output.txt
index 93b652894..a48a58fba 100644
--- a/acceptance/bundle/variables/prepend-workspace-var/output.txt
+++ b/acceptance/bundle/variables/prepend-workspace-var/output.txt
@@ -7,7 +7,7 @@
     },
     "target": "dev",
     "terraform": {
-      "exec_path": "$TERRAFORM"
+      "exec_path": "[TERRAFORM]"
     }
   },
   "resources": {
@@ -15,7 +15,7 @@
       "my_job": {
         "deployment": {
           "kind": "BUNDLE",
-          "metadata_file_path": "/Users/$USERNAME/path/to/root/state/metadata.json"
+          "metadata_file_path": "/Users/[USERNAME]/path/to/root/state/metadata.json"
         },
         "edit_mode": "UI_LOCKED",
         "format": "MULTI_TASK",
@@ -29,7 +29,7 @@
             "existing_cluster_id": "500",
             "python_wheel_task": {
               "named_parameters": {
-                "conf-file": "/Users/$USERNAME/path/to/root/files/path/to/config.yaml"
+                "conf-file": "/Users/[USERNAME]/path/to/root/files/path/to/config.yaml"
               }
             },
             "task_key": ""
@@ -46,17 +46,17 @@
   "targets": null,
   "variables": {
     "workspace_root": {
-      "default": "/Users/$USERNAME",
+      "default": "/Users/[USERNAME]",
       "description": "root directory in the Databricks workspace to store the asset bundle and associated artifacts",
-      "value": "/Users/$USERNAME"
+      "value": "/Users/[USERNAME]"
     }
   },
   "workspace": {
-    "artifact_path": "/Users/$USERNAME/path/to/root/artifacts",
-    "file_path": "/Users/$USERNAME/path/to/root/files",
+    "artifact_path": "/Users/[USERNAME]/path/to/root/artifacts",
+    "file_path": "/Users/[USERNAME]/path/to/root/files",
     "profile": "profile_name",
-    "resource_path": "/Users/$USERNAME/path/to/root/resources",
-    "root_path": "/Users/$USERNAME/path/to/root",
-    "state_path": "/Users/$USERNAME/path/to/root/state"
+    "resource_path": "/Users/[USERNAME]/path/to/root/resources",
+    "root_path": "/Users/[USERNAME]/path/to/root",
+    "state_path": "/Users/[USERNAME]/path/to/root/state"
   }
 }
diff --git a/acceptance/bundle/variables/resolve-nonstrings/output.txt b/acceptance/bundle/variables/resolve-nonstrings/output.txt
index 3a1eb9c47..951ad7a0d 100644
--- a/acceptance/bundle/variables/resolve-nonstrings/output.txt
+++ b/acceptance/bundle/variables/resolve-nonstrings/output.txt
@@ -20,7 +20,7 @@
     "job1": {
       "deployment": {
         "kind": "BUNDLE",
-        "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/TestResolveVariableReferencesForPrimitiveNonStringFields/default/state/metadata.json"
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/TestResolveVariableReferencesForPrimitiveNonStringFields/default/state/metadata.json"
       },
       "edit_mode": "UI_LOCKED",
       "format": "MULTI_TASK",
diff --git a/acceptance/bundle/variables/vanilla/output.txt b/acceptance/bundle/variables/vanilla/output.txt
index e98882bb0..3958c39b9 100644
--- a/acceptance/bundle/variables/vanilla/output.txt
+++ b/acceptance/bundle/variables/vanilla/output.txt
@@ -1,15 +1,15 @@
 
->>> BUNDLE_VAR_b=def $CLI bundle validate -o json
+>>> BUNDLE_VAR_b=def [CLI] bundle validate -o json
 "abc def"
 
->>> errcode $CLI bundle validate
+>>> errcode [CLI] bundle validate
 Error: no value assigned to required variable b. Assignment can be done using "--var", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle/<target>/variable-overrides.json file
 
 Name: ${var.a} ${var.b}
 Target: default
 Workspace:
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/${var.a} ${var.b}/default
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/${var.a} ${var.b}/default
 
 Found 1 error
 
diff --git a/acceptance/bundle/variables/variable_overrides_in_target/output.txt b/acceptance/bundle/variables/variable_overrides_in_target/output.txt
index 8998b691d..d112cf2de 100644
--- a/acceptance/bundle/variables/variable_overrides_in_target/output.txt
+++ b/acceptance/bundle/variables/variable_overrides_in_target/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI bundle validate -o json -t use-default-variable-values
+>>> [CLI] bundle validate -o json -t use-default-variable-values
 {
   "pipelines": {
     "my_pipeline": {
@@ -12,7 +12,7 @@
       "continuous": true,
       "deployment": {
         "kind": "BUNDLE",
-        "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/foobar/use-default-variable-values/state/metadata.json"
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/foobar/use-default-variable-values/state/metadata.json"
       },
       "name": "a_string",
       "permissions": []
@@ -20,7 +20,7 @@
   }
 }
 
->>> $CLI bundle validate -o json -t override-string-variable
+>>> [CLI] bundle validate -o json -t override-string-variable
 {
   "pipelines": {
     "my_pipeline": {
@@ -33,7 +33,7 @@
       "continuous": true,
       "deployment": {
         "kind": "BUNDLE",
-        "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/foobar/override-string-variable/state/metadata.json"
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/foobar/override-string-variable/state/metadata.json"
       },
       "name": "overridden_string",
       "permissions": []
@@ -41,7 +41,7 @@
   }
 }
 
->>> $CLI bundle validate -o json -t override-int-variable
+>>> [CLI] bundle validate -o json -t override-int-variable
 {
   "pipelines": {
     "my_pipeline": {
@@ -54,7 +54,7 @@
       "continuous": true,
       "deployment": {
         "kind": "BUNDLE",
-        "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/foobar/override-int-variable/state/metadata.json"
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/foobar/override-int-variable/state/metadata.json"
       },
       "name": "a_string",
       "permissions": []
@@ -62,7 +62,7 @@
   }
 }
 
->>> $CLI bundle validate -o json -t override-both-bool-and-string-variables
+>>> [CLI] bundle validate -o json -t override-both-bool-and-string-variables
 {
   "pipelines": {
     "my_pipeline": {
@@ -75,7 +75,7 @@
       "continuous": false,
       "deployment": {
         "kind": "BUNDLE",
-        "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/foobar/override-both-bool-and-string-variables/state/metadata.json"
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/foobar/override-both-bool-and-string-variables/state/metadata.json"
       },
       "name": "overridden_string",
       "permissions": []
diff --git a/acceptance/selftest/output.txt b/acceptance/selftest/output.txt
index 91aa8c33e..cadbdebb5 100644
--- a/acceptance/selftest/output.txt
+++ b/acceptance/selftest/output.txt
@@ -18,13 +18,13 @@ Exit code: 7
 
 === Capturing pwd
 >>> python3 -c import os; print(os.getcwd())
-$TMPDIR
+[TMPDIR]
 
 === Capturing subdir
 >>> mkdir -p subdir/a/b/c
 
 >>> withdir subdir/a/b/c python3 -c import os; print(os.getcwd())
-$TMPDIR/subdir/a/b/c
+[TMPDIR]/subdir/a/b/c
 
 === Custom output files - everything starting with out is captured and compared
 >>> echo HELLO
@@ -35,5 +35,5 @@ CUSTOM_NUMBER_REGEX
 123456
 
 === Testing --version
->>> $CLI --version
-Databricks CLI v$DEV_VERSION
+>>> [CLI] --version
+Databricks CLI v[DEV_VERSION]
diff --git a/acceptance/selftest/test.toml b/acceptance/selftest/test.toml
index 9607ec5df..762e28ceb 100644
--- a/acceptance/selftest/test.toml
+++ b/acceptance/selftest/test.toml
@@ -16,5 +16,5 @@ New = "CUSTOM_NUMBER_REGEX"
 
 [[Repls]]
 # Fix path with reverse slashes in the output for Windows.
-Old = '\$TMPDIR\\subdir\\a\\b\\c'
-New = '$$TMPDIR/subdir/a/b/c'
+Old = 'TMPDIR]\\subdir\\a\\b\\c'
+New = 'TMPDIR]/subdir/a/b/c'
diff --git a/acceptance/terraform/output.txt b/acceptance/terraform/output.txt
index 32589ddab..6bdc809f6 100644
--- a/acceptance/terraform/output.txt
+++ b/acceptance/terraform/output.txt
@@ -1,5 +1,5 @@
 
->>> $TERRAFORM init -no-color -get=false
+>>> [TERRAFORM] init -no-color -get=false
 
 Initializing the backend...
 
@@ -35,12 +35,12 @@ If you ever set or change modules or backend configuration for Terraform,
 rerun this command to reinitialize your working directory. If you forget, other
 commands will detect it and remind you to do so if necessary.
 
->>> $TERRAFORM plan -no-color
+>>> [TERRAFORM] plan -no-color
 data.databricks_current_user.me: Reading...
-data.databricks_current_user.me: Read complete after (redacted) [id=$USER.Id]
+data.databricks_current_user.me: Read complete after (redacted) [id=[USERID]]
 
 Changes to Outputs:
-  + username = "$USERNAME"
+  + username = "[USERNAME]"
 
 You can apply this plan to save these new output values to the Terraform
 state, without changing any real infrastructure.
diff --git a/acceptance/workspace/jobs/create/output.txt b/acceptance/workspace/jobs/create/output.txt
index a9487fe5b..50b823aa0 100644
--- a/acceptance/workspace/jobs/create/output.txt
+++ b/acceptance/workspace/jobs/create/output.txt
@@ -1,5 +1,5 @@
 
->>> $CLI jobs create --json {"name":"abc"}
+>>> [CLI] jobs create --json {"name":"abc"}
 {
   "job_id":1111
 }
diff --git a/integration/bundle/testdata/apps/bundle_deploy.txt b/integration/bundle/testdata/apps/bundle_deploy.txt
index 211164174..437a55596 100644
--- a/integration/bundle/testdata/apps/bundle_deploy.txt
+++ b/integration/bundle/testdata/apps/bundle_deploy.txt
@@ -1,4 +1,4 @@
-Uploading bundle files to /Workspace/Users/$USERNAME/.bundle/$UNIQUE_PRJ/files...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/$UNIQUE_PRJ/files...
 Deploying resources...
 Updating deployment state...
 Deployment complete!
diff --git a/integration/bundle/testdata/apps/bundle_validate.txt b/integration/bundle/testdata/apps/bundle_validate.txt
index dc9016a0f..567fafd24 100644
--- a/integration/bundle/testdata/apps/bundle_validate.txt
+++ b/integration/bundle/testdata/apps/bundle_validate.txt
@@ -1,7 +1,7 @@
 Name: basic
 Target: default
 Workspace:
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/$UNIQUE_PRJ
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/$UNIQUE_PRJ
 
 Validation OK!
diff --git a/integration/bundle/testdata/default_python/bundle_deploy.txt b/integration/bundle/testdata/default_python/bundle_deploy.txt
index d7b8cede9..076e7618f 100644
--- a/integration/bundle/testdata/default_python/bundle_deploy.txt
+++ b/integration/bundle/testdata/default_python/bundle_deploy.txt
@@ -1,6 +1,6 @@
 Building project_name_$UNIQUE_PRJ...
 Uploading project_name_$UNIQUE_PRJ-0.0.1+[NUMID].[NUMID]-py3-none-any.whl...
-Uploading bundle files to /Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/files...
 Deploying resources...
 Updating deployment state...
 Deployment complete!
diff --git a/integration/bundle/testdata/default_python/bundle_init.txt b/integration/bundle/testdata/default_python/bundle_init.txt
index c2917ea4e..6ea0801ad 100644
--- a/integration/bundle/testdata/default_python/bundle_init.txt
+++ b/integration/bundle/testdata/default_python/bundle_init.txt
@@ -1,6 +1,6 @@
 
 Welcome to the default Python template for Databricks Asset Bundles!
-Workspace to use (auto-detected, edit in 'project_name_$UNIQUE_PRJ/databricks.yml'): $DATABRICKS_URL
+Workspace to use (auto-detected, edit in 'project_name_$UNIQUE_PRJ/databricks.yml'): [DATABRICKS_URL]
 
 ✨ Your new project has been created in the 'project_name_$UNIQUE_PRJ' directory!
 
diff --git a/integration/bundle/testdata/default_python/bundle_summary.txt b/integration/bundle/testdata/default_python/bundle_summary.txt
index 0b4c15764..450f01c46 100644
--- a/integration/bundle/testdata/default_python/bundle_summary.txt
+++ b/integration/bundle/testdata/default_python/bundle_summary.txt
@@ -22,54 +22,54 @@
     "resources/project_name_$UNIQUE_PRJ.pipeline.yml"
   ],
   "workspace": {
-    "host": "$DATABRICKS_URL",
+    "host": "[DATABRICKS_URL]",
     "current_user": {
       "active": true,
-      "displayName": "$USERNAME",
+      "displayName": "[USERNAME]",
       "emails": [
         {
           "primary": true,
           "type": "work",
-          "value": "$USERNAME"
+          "value": "[USERNAME]"
         }
       ],
       "groups": [
         {
-          "$ref": "Groups/$USER.Groups[0]",
+          "$ref": "Groups/[USERGROUP]",
           "display": "team.engineering",
           "type": "direct",
-          "value": "$USER.Groups[0]"
+          "value": "[USERGROUP]"
         }
       ],
-      "id": "$USER.Id",
+      "id": "[USERID]",
       "name": {
-        "familyName": "$USERNAME",
-        "givenName": "$USERNAME"
+        "familyName": "[USERNAME]",
+        "givenName": "[USERNAME]"
       },
       "schemas": [
         "urn:ietf:params:scim:schemas:core:2.0:User",
         "urn:ietf:params:scim:schemas:extension:workspace:2.0:User"
       ],
-      "short_name": "$USERNAME",
-      "userName": "$USERNAME"
+      "short_name": "[USERNAME]",
+      "userName": "[USERNAME]"
     },
-    "root_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev",
-    "file_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files",
-    "resource_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/resources",
-    "artifact_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/artifacts",
-    "state_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/state"
+    "root_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev",
+    "file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/files",
+    "resource_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/resources",
+    "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/artifacts",
+    "state_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/state"
   },
   "resources": {
     "jobs": {
       "project_name_$UNIQUE_PRJ_job": {
         "deployment": {
           "kind": "BUNDLE",
-          "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/state/metadata.json"
+          "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/state/metadata.json"
         },
         "edit_mode": "UI_LOCKED",
         "email_notifications": {
           "on_failure": [
-            "$USERNAME"
+            "[USERNAME]"
           ]
         },
         "format": "MULTI_TASK",
@@ -88,18 +88,18 @@
           }
         ],
         "max_concurrent_runs": 4,
-        "name": "[dev $USERNAME] project_name_$UNIQUE_PRJ_job",
+        "name": "[dev [USERNAME]] project_name_$UNIQUE_PRJ_job",
         "queue": {
           "enabled": true
         },
         "tags": {
-          "dev": "$USERNAME"
+          "dev": "[USERNAME]"
         },
         "tasks": [
           {
             "job_cluster_key": "job_cluster",
             "notebook_task": {
-              "notebook_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files/src/notebook"
+              "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/files/src/notebook"
             },
             "task_key": "notebook_task"
           },
@@ -140,31 +140,31 @@
             "unit": "DAYS"
           }
         },
-        "url": "$DATABRICKS_URL/jobs/[NUMID]?o=[NUMID]"
+        "url": "[DATABRICKS_URL]/jobs/[NUMID]?o=[NUMID]"
       }
     },
     "pipelines": {
       "project_name_$UNIQUE_PRJ_pipeline": {
         "catalog": "main",
         "configuration": {
-          "bundle.sourcePath": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files/src"
+          "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/files/src"
         },
         "deployment": {
           "kind": "BUNDLE",
-          "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/state/metadata.json"
+          "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/state/metadata.json"
         },
         "development": true,
         "id": "[UUID]",
         "libraries": [
           {
             "notebook": {
-              "path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files/src/dlt_pipeline"
+              "path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/files/src/dlt_pipeline"
             }
           }
         ],
-        "name": "[dev $USERNAME] project_name_$UNIQUE_PRJ_pipeline",
+        "name": "[dev [USERNAME]] project_name_$UNIQUE_PRJ_pipeline",
         "target": "project_name_$UNIQUE_PRJ_dev",
-        "url": "$DATABRICKS_URL/pipelines/[UUID]?o=[NUMID]"
+        "url": "[DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID]"
       }
     }
   },
@@ -174,12 +174,12 @@
     ]
   },
   "presets": {
-    "name_prefix": "[dev $USERNAME] ",
+    "name_prefix": "[dev [USERNAME]] ",
     "pipelines_development": true,
     "trigger_pause_status": "PAUSED",
     "jobs_max_concurrent_runs": 4,
     "tags": {
-      "dev": "$USERNAME"
+      "dev": "[USERNAME]"
     }
   }
-}
+}
\ No newline at end of file
diff --git a/integration/bundle/testdata/default_python/bundle_validate.txt b/integration/bundle/testdata/default_python/bundle_validate.txt
index 578fd6494..c5c62b521 100644
--- a/integration/bundle/testdata/default_python/bundle_validate.txt
+++ b/integration/bundle/testdata/default_python/bundle_validate.txt
@@ -1,8 +1,8 @@
 Name: project_name_$UNIQUE_PRJ
 Target: dev
 Workspace:
-  Host: $DATABRICKS_URL
-  User: $USERNAME
-  Path: /Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev
+  Host: [DATABRICKS_URL]
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev
 
 Validation OK!
diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go
index 7077e611b..5bbba1be1 100644
--- a/libs/testdiff/replacement.go
+++ b/libs/testdiff/replacement.go
@@ -2,7 +2,6 @@ package testdiff
 
 import (
 	"encoding/json"
-	"fmt"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -16,7 +15,7 @@ import (
 )
 
 const (
-	testerName = "$USERNAME"
+	testerName = "[USERNAME]"
 )
 
 var (
@@ -140,25 +139,25 @@ func PrepareReplacementsWorkspaceClient(t testutil.TestingT, r *ReplacementsCont
 	t.Helper()
 	// in some clouds (gcp) w.Config.Host includes "https://" prefix in others it's really just a host (azure)
 	host := strings.TrimPrefix(strings.TrimPrefix(w.Config.Host, "http://"), "https://")
-	r.Set("https://"+host, "$DATABRICKS_URL")
-	r.Set("http://"+host, "$DATABRICKS_URL")
-	r.Set(host, "$DATABRICKS_HOST")
-	r.Set(w.Config.ClusterID, "$DATABRICKS_CLUSTER_ID")
-	r.Set(w.Config.WarehouseID, "$DATABRICKS_WAREHOUSE_ID")
-	r.Set(w.Config.ServerlessComputeID, "$DATABRICKS_SERVERLESS_COMPUTE_ID")
-	r.Set(w.Config.AccountID, "$DATABRICKS_ACCOUNT_ID")
-	r.Set(w.Config.Username, "$DATABRICKS_USERNAME")
-	r.SetPath(w.Config.Profile, "$DATABRICKS_CONFIG_PROFILE")
-	r.Set(w.Config.ConfigFile, "$DATABRICKS_CONFIG_FILE")
-	r.Set(w.Config.GoogleServiceAccount, "$DATABRICKS_GOOGLE_SERVICE_ACCOUNT")
-	r.Set(w.Config.AzureResourceID, "$DATABRICKS_AZURE_RESOURCE_ID")
+	r.Set("https://"+host, "[DATABRICKS_URL]")
+	r.Set("http://"+host, "[DATABRICKS_URL]")
+	r.Set(host, "[DATABRICKS_HOST]")
+	r.Set(w.Config.ClusterID, "[DATABRICKS_CLUSTER_ID]")
+	r.Set(w.Config.WarehouseID, "[DATABRICKS_WAREHOUSE_ID]")
+	r.Set(w.Config.ServerlessComputeID, "[DATABRICKS_SERVERLESS_COMPUTE_ID]")
+	r.Set(w.Config.AccountID, "[DATABRICKS_ACCOUNT_ID]")
+	r.Set(w.Config.Username, "[DATABRICKS_USERNAME]")
+	r.SetPath(w.Config.Profile, "[DATABRICKS_CONFIG_PROFILE]")
+	r.Set(w.Config.ConfigFile, "[DATABRICKS_CONFIG_FILE]")
+	r.Set(w.Config.GoogleServiceAccount, "[DATABRICKS_GOOGLE_SERVICE_ACCOUNT]")
+	r.Set(w.Config.AzureResourceID, "[DATABRICKS_AZURE_RESOURCE_ID]")
 	r.Set(w.Config.AzureClientID, testerName)
-	r.Set(w.Config.AzureTenantID, "$ARM_TENANT_ID")
-	r.Set(w.Config.AzureEnvironment, "$ARM_ENVIRONMENT")
-	r.Set(w.Config.ClientID, "$DATABRICKS_CLIENT_ID")
-	r.SetPath(w.Config.DatabricksCliPath, "$DATABRICKS_CLI_PATH")
+	r.Set(w.Config.AzureTenantID, "[ARM_TENANT_ID]")
+	r.Set(w.Config.AzureEnvironment, "[ARM_ENVIRONMENT]")
+	r.Set(w.Config.ClientID, "[DATABRICKS_CLIENT_ID]")
+	r.SetPath(w.Config.DatabricksCliPath, "[DATABRICKS_CLI_PATH]")
 	// This is set to words like "path" that happen too frequently
-	// r.Set(w.Config.AuthType, "$DATABRICKS_AUTH_TYPE")
+	// r.Set(w.Config.AuthType, "[DATABRICKS_AUTH_TYPE]")
 }
 
 func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam.User) {
@@ -179,14 +178,14 @@ func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam.
 
 	r.Set(iamutil.GetShortUserName(&u), testerName)
 
-	for ind, val := range u.Groups {
-		r.Set(val.Value, fmt.Sprintf("$USER.Groups[%d]", ind))
+	for _, val := range u.Groups {
+		r.Set(val.Value, "[USERGROUP]")
 	}
 
-	r.Set(u.Id, "$USER.Id")
+	r.Set(u.Id, "[USERID]")
 
-	for ind, val := range u.Roles {
-		r.Set(val.Value, fmt.Sprintf("$USER.Roles[%d]", ind))
+	for _, val := range u.Roles {
+		r.Set(val.Value, "[USERROLE]")
 	}
 }
 
@@ -207,5 +206,5 @@ func PrepareReplacementsTemporaryDirectory(t testutil.TestingT, r *ReplacementsC
 
 func PrepareReplacementsDevVersion(t testutil.TestingT, r *ReplacementsContext) {
 	t.Helper()
-	r.append(devVersionRegex, "$$DEV_VERSION")
+	r.append(devVersionRegex, "[DEV_VERSION]")
 }

From 2eb9abb5ee5db2984de5d791f1d347f7c91e5e21 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 3 Feb 2025 17:00:52 +0100
Subject: [PATCH 191/247] Bump github.com/spf13/pflag from 1.0.5 to 1.0.6
 (#2281)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bumps [github.com/spf13/pflag](https://github.com/spf13/pflag) from
1.0.5 to 1.0.6.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/spf13/pflag/releases">github.com/spf13/pflag's
releases</a>.</em></p>
<blockquote>
<h2>v1.0.6</h2>
<h2>What's Changed</h2>
<ul>
<li>Add exported functions to preserve <code>pkg/flag</code>
compatibility by <a
href="https://github.com/mckern"><code>@​mckern</code></a> in <a
href="https://redirect.github.com/spf13/pflag/pull/220">spf13/pflag#220</a></li>
<li>remove dead code for checking error nil by <a
href="https://github.com/yashbhutwala"><code>@​yashbhutwala</code></a>
in <a
href="https://redirect.github.com/spf13/pflag/pull/282">spf13/pflag#282</a></li>
<li>Add IPNetSlice and unit tests by <a
href="https://github.com/rpothier"><code>@​rpothier</code></a> in <a
href="https://redirect.github.com/spf13/pflag/pull/170">spf13/pflag#170</a></li>
<li>allow for blank ip addresses by <a
href="https://github.com/duhruh"><code>@​duhruh</code></a> in <a
href="https://redirect.github.com/spf13/pflag/pull/316">spf13/pflag#316</a></li>
<li>add github actions by <a
href="https://github.com/sagikazarmark"><code>@​sagikazarmark</code></a>
in <a
href="https://redirect.github.com/spf13/pflag/pull/419">spf13/pflag#419</a></li>
</ul>
<h2>New Contributors</h2>
<ul>
<li><a href="https://github.com/mckern"><code>@​mckern</code></a> made
their first contribution in <a
href="https://redirect.github.com/spf13/pflag/pull/220">spf13/pflag#220</a></li>
<li><a
href="https://github.com/yashbhutwala"><code>@​yashbhutwala</code></a>
made their first contribution in <a
href="https://redirect.github.com/spf13/pflag/pull/282">spf13/pflag#282</a></li>
<li><a href="https://github.com/rpothier"><code>@​rpothier</code></a>
made their first contribution in <a
href="https://redirect.github.com/spf13/pflag/pull/170">spf13/pflag#170</a></li>
<li><a href="https://github.com/duhruh"><code>@​duhruh</code></a> made
their first contribution in <a
href="https://redirect.github.com/spf13/pflag/pull/316">spf13/pflag#316</a></li>
<li><a
href="https://github.com/sagikazarmark"><code>@​sagikazarmark</code></a>
made their first contribution in <a
href="https://redirect.github.com/spf13/pflag/pull/419">spf13/pflag#419</a></li>
</ul>
<p><strong>Full Changelog</strong>: <a
href="https://github.com/spf13/pflag/compare/v1.0.5...v1.0.6">https://github.com/spf13/pflag/compare/v1.0.5...v1.0.6</a></p>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/spf13/pflag/commit/5ca813443bd2a4d9f46a253ea0407d23b3790713"><code>5ca8134</code></a>
Merge pull request <a
href="https://redirect.github.com/spf13/pflag/issues/419">#419</a> from
spf13/ci</li>
<li><a
href="https://github.com/spf13/pflag/commit/100ab0eb250792014fc1594d94f7fb5c5f0dee37"><code>100ab0e</code></a>
disable unsupported dependency graph for now</li>
<li><a
href="https://github.com/spf13/pflag/commit/a0f4ddd9fe01ac8fece07be6e3f3ae5dcd7ceec0"><code>a0f4ddd</code></a>
fix govet</li>
<li><a
href="https://github.com/spf13/pflag/commit/f48cbd1964b57ff7c17e2f233feb49c03efe6417"><code>f48cbd1</code></a>
add github actions</li>
<li><a
href="https://github.com/spf13/pflag/commit/d5e0c0615acee7028e1e2740a11102313be88de1"><code>d5e0c06</code></a>
allow for blank ip addresses (<a
href="https://redirect.github.com/spf13/pflag/issues/316">#316</a>)</li>
<li><a
href="https://github.com/spf13/pflag/commit/85dd5c8bc61cfa382fecd072378089d4e856579d"><code>85dd5c8</code></a>
Add IPNetSlice and unit tests (<a
href="https://redirect.github.com/spf13/pflag/issues/170">#170</a>)</li>
<li><a
href="https://github.com/spf13/pflag/commit/6971c29c4a22981adeaee7f4b437c0cffe08c031"><code>6971c29</code></a>
remove dead code for checking error nil (<a
href="https://redirect.github.com/spf13/pflag/issues/282">#282</a>)</li>
<li><a
href="https://github.com/spf13/pflag/commit/81378bbcd8a1005f72b1e8d7579e5dd7b2d612ab"><code>81378bb</code></a>
Add exported functions to preserve <code>pkg/flag</code> compatibility
(<a
href="https://redirect.github.com/spf13/pflag/issues/220">#220</a>)</li>
<li>See full diff in <a
href="https://github.com/spf13/pflag/compare/v1.0.5...v1.0.6">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/spf13/pflag&package-manager=go_modules&previous-version=1.0.5&new-version=1.0.6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/go.mod b/go.mod
index 151133944..b3f11e918 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // BSD-2-Clause
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // MIT
 	github.com/spf13/cobra v1.8.1 // Apache 2.0
-	github.com/spf13/pflag v1.0.5 // BSD-3-Clause
+	github.com/spf13/pflag v1.0.6 // BSD-3-Clause
 	github.com/stretchr/testify v1.10.0 // MIT
 	github.com/wI2L/jsondiff v0.6.1 // MIT
 	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
diff --git a/go.sum b/go.sum
index 3c7f20937..4e295a82d 100644
--- a/go.sum
+++ b/go.sum
@@ -147,8 +147,9 @@ github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0
 github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M=
 github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
 github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
+github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=

From 07efe830231d0b2c43079faeb1daee30781eb6cc Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 4 Feb 2025 17:08:01 +0100
Subject: [PATCH 192/247] Use go-version-file instead of go-version in  github
 actions (#2290)

This minimizes number of places where we hard-code go version.

Note, since we have go version specified without patch version ("1.23")
in go.mod, it will use most recent in 1.23.x line. I think this is fine.


https://github.com/actions/setup-go?tab=readme-ov-file#getting-go-version-from-the-gomod-file
---
 .github/workflows/push.yml             | 6 +++---
 .github/workflows/release-snapshot.yml | 2 +-
 .github/workflows/release.yml          | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
index f27459baa..c41afc18c 100644
--- a/.github/workflows/push.yml
+++ b/.github/workflows/push.yml
@@ -50,7 +50,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
-          go-version: 1.23.4
+          go-version-file: go.mod
 
       - name: Setup Python
         uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
@@ -82,7 +82,7 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
-          go-version: 1.23.4
+          go-version-file: go.mod
           # Use different schema from regular job, to avoid overwriting the same key
           cache-dependency-path: |
             go.sum
@@ -116,7 +116,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
-          go-version: 1.23.4
+          go-version-file: go.mod
           # Use different schema from regular job, to avoid overwriting the same key
           cache-dependency-path: |
             go.sum
diff --git a/.github/workflows/release-snapshot.yml b/.github/workflows/release-snapshot.yml
index 548d93e90..8b4684eab 100644
--- a/.github/workflows/release-snapshot.yml
+++ b/.github/workflows/release-snapshot.yml
@@ -34,7 +34,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
-          go-version: 1.23.4
+          go-version-file: go.mod
 
           # The default cache key for this action considers only the `go.sum` file.
           # We include .goreleaser.yaml here to differentiate from the cache used by the push action
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 5d5811b19..fe5b4170b 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -26,7 +26,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
-          go-version: 1.23.4
+          go-version-file: go.mod
 
           # The default cache key for this action considers only the `go.sum` file.
           # We include .goreleaser.yaml here to differentiate from the cache used by the push action

From d86ad9189902d629cc85e0d79250a7301bb29aeb Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Tue, 4 Feb 2025 22:08:11 +0530
Subject: [PATCH 193/247] Allow test servers to return errors responses (#2291)

## Changes
The APIs at Databricks when returning a non `200` status code will
return a response body of the format:
```
{
  "error_code": "Error code",
  "message": "Human-readable error message."
}
```

This PR adds the ability to stub non-200 status codes in the test
server, allowing us to mock API errors from Databricks.
## Tests
New test
---
 acceptance/acceptance_test.go                 |  8 +++--
 acceptance/cmd_server_test.go                 |  4 +--
 acceptance/config_test.go                     |  3 +-
 acceptance/server_test.go                     | 32 +++++++++----------
 .../jobs/create-error/out.requests.txt        |  1 +
 .../workspace/jobs/create-error/output.txt    |  5 +++
 acceptance/workspace/jobs/create-error/script |  1 +
 .../workspace/jobs/create-error/test.toml     | 12 +++++++
 libs/testserver/server.go                     | 11 +++----
 9 files changed, 49 insertions(+), 28 deletions(-)
 create mode 100644 acceptance/workspace/jobs/create-error/out.requests.txt
 create mode 100644 acceptance/workspace/jobs/create-error/output.txt
 create mode 100644 acceptance/workspace/jobs/create-error/script
 create mode 100644 acceptance/workspace/jobs/create-error/test.toml

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 871b8bd62..6e302fa96 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -261,8 +261,12 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 
 		for _, stub := range config.Server {
 			require.NotEmpty(t, stub.Pattern)
-			server.Handle(stub.Pattern, func(req *http.Request) (resp any, err error) {
-				return stub.Response.Body, nil
+			server.Handle(stub.Pattern, func(req *http.Request) (any, int) {
+				statusCode := http.StatusOK
+				if stub.Response.StatusCode != 0 {
+					statusCode = stub.Response.StatusCode
+				}
+				return stub.Response.Body, statusCode
 			})
 		}
 		cmd.Env = append(cmd.Env, "DATABRICKS_HOST="+server.URL)
diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go
index 9af63d0db..04d56c7d4 100644
--- a/acceptance/cmd_server_test.go
+++ b/acceptance/cmd_server_test.go
@@ -15,7 +15,7 @@ import (
 func StartCmdServer(t *testing.T) *testserver.Server {
 	server := testserver.New(t)
 
-	server.Handle("/", func(r *http.Request) (any, error) {
+	server.Handle("/", func(r *http.Request) (any, int) {
 		q := r.URL.Query()
 		args := strings.Split(q.Get("args"), " ")
 
@@ -40,7 +40,7 @@ func StartCmdServer(t *testing.T) *testserver.Server {
 			exitcode = 1
 		}
 		result["exitcode"] = exitcode
-		return result, nil
+		return result, http.StatusOK
 	})
 	return server
 }
diff --git a/acceptance/config_test.go b/acceptance/config_test.go
index c7be223de..97be457e2 100644
--- a/acceptance/config_test.go
+++ b/acceptance/config_test.go
@@ -57,7 +57,8 @@ type ServerStub struct {
 
 	// The response body to return.
 	Response struct {
-		Body string
+		Body       string
+		StatusCode int
 	}
 }
 
diff --git a/acceptance/server_test.go b/acceptance/server_test.go
index 4957a7668..a7695b21e 100644
--- a/acceptance/server_test.go
+++ b/acceptance/server_test.go
@@ -11,7 +11,7 @@ import (
 )
 
 func AddHandlers(server *testserver.Server) {
-	server.Handle("GET /api/2.0/policies/clusters/list", func(r *http.Request) (any, error) {
+	server.Handle("GET /api/2.0/policies/clusters/list", func(r *http.Request) (any, int) {
 		return compute.ListPoliciesResponse{
 			Policies: []compute.Policy{
 				{
@@ -23,10 +23,10 @@ func AddHandlers(server *testserver.Server) {
 					Name:     "some-test-cluster-policy",
 				},
 			},
-		}, nil
+		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.0/instance-pools/list", func(r *http.Request) (any, error) {
+	server.Handle("GET /api/2.0/instance-pools/list", func(r *http.Request) (any, int) {
 		return compute.ListInstancePools{
 			InstancePools: []compute.InstancePoolAndStats{
 				{
@@ -34,10 +34,10 @@ func AddHandlers(server *testserver.Server) {
 					InstancePoolId:   "1234",
 				},
 			},
-		}, nil
+		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.1/clusters/list", func(r *http.Request) (any, error) {
+	server.Handle("GET /api/2.1/clusters/list", func(r *http.Request) (any, int) {
 		return compute.ListClustersResponse{
 			Clusters: []compute.ClusterDetails{
 				{
@@ -49,32 +49,32 @@ func AddHandlers(server *testserver.Server) {
 					ClusterId:   "9876",
 				},
 			},
-		}, nil
+		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, error) {
+	server.Handle("GET /api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, int) {
 		return iam.User{
 			Id:       "1000012345",
 			UserName: "tester@databricks.com",
-		}, nil
+		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.0/workspace/get-status", func(r *http.Request) (any, error) {
+	server.Handle("GET /api/2.0/workspace/get-status", func(r *http.Request) (any, int) {
 		return workspace.ObjectInfo{
 			ObjectId:   1001,
 			ObjectType: "DIRECTORY",
 			Path:       "",
 			ResourceId: "1001",
-		}, nil
+		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.1/unity-catalog/current-metastore-assignment", func(r *http.Request) (any, error) {
+	server.Handle("GET /api/2.1/unity-catalog/current-metastore-assignment", func(r *http.Request) (any, int) {
 		return catalog.MetastoreAssignment{
 			DefaultCatalogName: "main",
-		}, nil
+		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.0/permissions/directories/1001", func(r *http.Request) (any, error) {
+	server.Handle("GET /api/2.0/permissions/directories/1001", func(r *http.Request) (any, int) {
 		return workspace.WorkspaceObjectPermissions{
 			ObjectId:   "1001",
 			ObjectType: "DIRECTORY",
@@ -88,10 +88,10 @@ func AddHandlers(server *testserver.Server) {
 					},
 				},
 			},
-		}, nil
+		}, http.StatusOK
 	})
 
-	server.Handle("POST /api/2.0/workspace/mkdirs", func(r *http.Request) (any, error) {
-		return "{}", nil
+	server.Handle("POST /api/2.0/workspace/mkdirs", func(r *http.Request) (any, int) {
+		return "{}", http.StatusOK
 	})
 }
diff --git a/acceptance/workspace/jobs/create-error/out.requests.txt b/acceptance/workspace/jobs/create-error/out.requests.txt
new file mode 100644
index 000000000..b22876b70
--- /dev/null
+++ b/acceptance/workspace/jobs/create-error/out.requests.txt
@@ -0,0 +1 @@
+{"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}}
diff --git a/acceptance/workspace/jobs/create-error/output.txt b/acceptance/workspace/jobs/create-error/output.txt
new file mode 100644
index 000000000..0e69eeb4b
--- /dev/null
+++ b/acceptance/workspace/jobs/create-error/output.txt
@@ -0,0 +1,5 @@
+
+>>> [CLI] jobs create --json {"name":"abc"}
+Error: Invalid access token.
+
+Exit code: 1
diff --git a/acceptance/workspace/jobs/create-error/script b/acceptance/workspace/jobs/create-error/script
new file mode 100644
index 000000000..9ff7b5b87
--- /dev/null
+++ b/acceptance/workspace/jobs/create-error/script
@@ -0,0 +1 @@
+trace $CLI jobs create --json '{"name":"abc"}'
diff --git a/acceptance/workspace/jobs/create-error/test.toml b/acceptance/workspace/jobs/create-error/test.toml
new file mode 100644
index 000000000..b45bf77e5
--- /dev/null
+++ b/acceptance/workspace/jobs/create-error/test.toml
@@ -0,0 +1,12 @@
+LocalOnly = true  # request recording currently does not work with cloud environment
+RecordRequests = true
+
+[[Server]]
+Pattern = "POST /api/2.1/jobs/create"
+Response.Body = '''
+{
+    "error_code": "PERMISSION_DENIED",
+    "message": "Invalid access token."
+}
+'''
+Response.StatusCode = 403
diff --git a/libs/testserver/server.go b/libs/testserver/server.go
index 2e8dbdfda..a751531ed 100644
--- a/libs/testserver/server.go
+++ b/libs/testserver/server.go
@@ -40,15 +40,11 @@ func New(t testutil.TestingT) *Server {
 	}
 }
 
-type HandlerFunc func(req *http.Request) (resp any, err error)
+type HandlerFunc func(req *http.Request) (resp any, statusCode int)
 
 func (s *Server) Handle(pattern string, handler HandlerFunc) {
 	s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) {
-		resp, err := handler(r)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
+		resp, statusCode := handler(r)
 
 		if s.RecordRequests {
 			body, err := io.ReadAll(r.Body)
@@ -63,9 +59,10 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) {
 		}
 
 		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(statusCode)
 
 		var respBytes []byte
-
+		var err error
 		respString, ok := resp.(string)
 		if ok {
 			respBytes = []byte(respString)

From 84b694f2a158186fa6d641e55e51af29761fa419 Mon Sep 17 00:00:00 2001
From: Simon Poltier <simon.poltier@mirai-solutions.com>
Date: Tue, 4 Feb 2025 19:28:19 +0100
Subject: [PATCH 194/247] accept JSON includes (#2265)

#2201 disabled using JSON as part of a bundle definition. I believe this
was not intended.

## Changes
Accept json files as includes, just as YAML files.
## Tests
Covered by the tests in #2201
---
 acceptance/bundle/includes/non_yaml_in_include/output.txt | 4 ++--
 bundle/config/loader/process_root_includes.go             | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/acceptance/bundle/includes/non_yaml_in_include/output.txt b/acceptance/bundle/includes/non_yaml_in_include/output.txt
index 6006ca14e..f5211cc4b 100644
--- a/acceptance/bundle/includes/non_yaml_in_include/output.txt
+++ b/acceptance/bundle/includes/non_yaml_in_include/output.txt
@@ -1,7 +1,7 @@
-Error: Files in the 'include' configuration section must be YAML files.
+Error: Files in the 'include' configuration section must be YAML or JSON files.
   in databricks.yml:5:4
 
-The file test.py in the 'include' configuration section is not a YAML file, and only YAML files are supported. To include files to sync, specify them in the 'sync.include' configuration section instead.
+The file test.py in the 'include' configuration section is not a YAML or JSON file, and only such files are supported. To include files to sync, specify them in the 'sync.include' configuration section instead.
 
 Name: non_yaml_in_includes
 
diff --git a/bundle/config/loader/process_root_includes.go b/bundle/config/loader/process_root_includes.go
index 198095742..69e6dd4e4 100644
--- a/bundle/config/loader/process_root_includes.go
+++ b/bundle/config/loader/process_root_includes.go
@@ -71,11 +71,11 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.
 				continue
 			}
 			seen[rel] = true
-			if filepath.Ext(rel) != ".yaml" && filepath.Ext(rel) != ".yml" {
+			if filepath.Ext(rel) != ".yaml" && filepath.Ext(rel) != ".yml" && filepath.Ext(rel) != ".json" {
 				diags = diags.Append(diag.Diagnostic{
 					Severity:  diag.Error,
-					Summary:   "Files in the 'include' configuration section must be YAML files.",
-					Detail:    fmt.Sprintf("The file %s in the 'include' configuration section is not a YAML file, and only YAML files are supported. To include files to sync, specify them in the 'sync.include' configuration section instead.", rel),
+					Summary:   "Files in the 'include' configuration section must be YAML or JSON files.",
+					Detail:    fmt.Sprintf("The file %s in the 'include' configuration section is not a YAML or JSON file, and only such files are supported. To include files to sync, specify them in the 'sync.include' configuration section instead.", rel),
 					Locations: b.Config.GetLocations(fmt.Sprintf("include[%d]", i)),
 				})
 				continue

From dcc61cd7636b4cdc647256bbd07aac8d4651da9e Mon Sep 17 00:00:00 2001
From: rikjansen-hu <rik.jansen@hu.nl>
Date: Tue, 4 Feb 2025 19:30:02 +0100
Subject: [PATCH 195/247] Fix env variable for AzureCli local config (#2248)

## Changes

Solves #1722 (current solution passes wrong variable)

## Tests

None, this is a simple find-and-replace on a previous PR.
Proof that this is the correct
[variable](https://learn.microsoft.com/en-us/cli/azure/azure-cli-configuration#cli-configuration-file).
This just passes the variable along to the Terraform environment, which
[should](https://github.com/hashicorp/terraform/issues/25416) be picked
up by Terraform.

Co-authored-by: Rik Jansen <rik.jansen2@nl.abnamro.com>
---
 bundle/deploy/terraform/init.go      | 4 ++--
 bundle/deploy/terraform/init_test.go | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/bundle/deploy/terraform/init.go b/bundle/deploy/terraform/init.go
index 5957611a4..a204222d0 100644
--- a/bundle/deploy/terraform/init.go
+++ b/bundle/deploy/terraform/init.go
@@ -101,9 +101,9 @@ var envCopy = []string{
 	// same auxiliary programs (e.g. `az`, or `gcloud`) as the CLI.
 	"PATH",
 
-	// Include $AZURE_CONFIG_FILE in set of environment variables to pass along.
+	// Include $AZURE_CONFIG_DIR in set of environment variables to pass along.
 	// This is set in Azure DevOps by the AzureCLI@2 task.
-	"AZURE_CONFIG_FILE",
+	"AZURE_CONFIG_DIR",
 
 	// Include $TF_CLI_CONFIG_FILE to override terraform provider in development.
 	// See: https://developer.hashicorp.com/terraform/cli/config/config-file#explicit-installation-method-configuration
diff --git a/bundle/deploy/terraform/init_test.go b/bundle/deploy/terraform/init_test.go
index c7a4ffe4a..4645ed007 100644
--- a/bundle/deploy/terraform/init_test.go
+++ b/bundle/deploy/terraform/init_test.go
@@ -292,7 +292,7 @@ func TestInheritEnvVars(t *testing.T) {
 	t.Setenv("HOME", "/home/testuser")
 	t.Setenv("PATH", "/foo:/bar")
 	t.Setenv("TF_CLI_CONFIG_FILE", "/tmp/config.tfrc")
-	t.Setenv("AZURE_CONFIG_FILE", "/tmp/foo/bar")
+	t.Setenv("AZURE_CONFIG_DIR", "/tmp/foo/bar")
 
 	ctx := context.Background()
 	env := map[string]string{}
@@ -301,7 +301,7 @@ func TestInheritEnvVars(t *testing.T) {
 		assert.Equal(t, "/home/testuser", env["HOME"])
 		assert.Equal(t, "/foo:/bar", env["PATH"])
 		assert.Equal(t, "/tmp/config.tfrc", env["TF_CLI_CONFIG_FILE"])
-		assert.Equal(t, "/tmp/foo/bar", env["AZURE_CONFIG_FILE"])
+		assert.Equal(t, "/tmp/foo/bar", env["AZURE_CONFIG_DIR"])
 	}
 }
 

From 2e1455841cced1337ad8ff8b456f57f217a9a74d Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Tue, 4 Feb 2025 22:20:02 +0100
Subject: [PATCH 196/247] Update CODEOWNERS for cmd/labs (#2295)

## Changes

The `CODEOWNERS` file must live in one of the directories specified in
the [docs][docs], so the existing file under `cmd/labs` didn't work.
This change moves the contents to the top-level file and includes
@alexott as owner.

[docs]:
https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#codeowners-file-location
---
 .github/CODEOWNERS  | 1 +
 cmd/labs/CODEOWNERS | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)
 delete mode 100644 cmd/labs/CODEOWNERS

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 76835de7d..3c3895bc1 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -1 +1,2 @@
 * @pietern @andrewnester @shreyas-goenka @denik
+cmd/labs @alexott @nfx
diff --git a/cmd/labs/CODEOWNERS b/cmd/labs/CODEOWNERS
deleted file mode 100644
index cc93a75e6..000000000
--- a/cmd/labs/CODEOWNERS
+++ /dev/null
@@ -1 +0,0 @@
-* @nfx

From 1678503cb04abfed7fdd1d595d93ed722c75330e Mon Sep 17 00:00:00 2001
From: Ilya Kuznetsov <ilya.kuznetsov@databricks.com>
Date: Wed, 5 Feb 2025 10:01:51 +0100
Subject: [PATCH 197/247] Fix docs template (#2283)

## Changes

Comment breaks markdown front-matter and description cannot be read

## Tests
<!-- How is this tested? -->
---
 bundle/docsgen/output/reference.md    | 3 ++-
 bundle/docsgen/output/resources.md    | 3 ++-
 bundle/docsgen/templates/reference.md | 3 ++-
 bundle/docsgen/templates/resources.md | 3 ++-
 4 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/bundle/docsgen/output/reference.md b/bundle/docsgen/output/reference.md
index a2241d017..8a89d354b 100644
--- a/bundle/docsgen/output/reference.md
+++ b/bundle/docsgen/output/reference.md
@@ -1,8 +1,9 @@
-<!-- DO NOT EDIT. This file is autogenerated with https://github.com/databricks/cli -->
 ---
 description: Configuration reference for databricks.yml
 ---
 
+<!-- DO NOT EDIT. This file is autogenerated with https://github.com/databricks/cli -->
+
 # Configuration reference
 
 This article provides reference for keys supported by <DABS> configuration (YAML). See [_](/dev-tools/bundles/index.md).
diff --git a/bundle/docsgen/output/resources.md b/bundle/docsgen/output/resources.md
index ff80ee635..df7578c73 100644
--- a/bundle/docsgen/output/resources.md
+++ b/bundle/docsgen/output/resources.md
@@ -1,8 +1,9 @@
-<!-- DO NOT EDIT. This file is autogenerated with https://github.com/databricks/cli -->
 ---
 description: Learn about resources supported by Databricks Asset Bundles and how to configure them.
 ---
 
+<!-- DO NOT EDIT. This file is autogenerated with https://github.com/databricks/cli -->
+
 # <DABS> resources
 
 <DABS> allows you to specify information about the <Databricks> resources used by the bundle in the `resources` mapping in the bundle configuration. See [resources mapping](/dev-tools/bundles/settings.md#resources) and [resources key reference](/dev-tools/bundles/reference.md#resources).
diff --git a/bundle/docsgen/templates/reference.md b/bundle/docsgen/templates/reference.md
index a17d53315..345afc509 100644
--- a/bundle/docsgen/templates/reference.md
+++ b/bundle/docsgen/templates/reference.md
@@ -1,8 +1,9 @@
-<!-- DO NOT EDIT. This file is autogenerated with https://github.com/databricks/cli -->
 ---
 description: Configuration reference for databricks.yml
 ---
 
+<!-- DO NOT EDIT. This file is autogenerated with https://github.com/databricks/cli -->
+
 # Configuration reference
 
 This article provides reference for keys supported by <DABS> configuration (YAML). See [_](/dev-tools/bundles/index.md).
diff --git a/bundle/docsgen/templates/resources.md b/bundle/docsgen/templates/resources.md
index fccfac47d..e9a6c8c5b 100644
--- a/bundle/docsgen/templates/resources.md
+++ b/bundle/docsgen/templates/resources.md
@@ -1,8 +1,9 @@
-<!-- DO NOT EDIT. This file is autogenerated with https://github.com/databricks/cli -->
 ---
 description: Learn about resources supported by Databricks Asset Bundles and how to configure them.
 ---
 
+<!-- DO NOT EDIT. This file is autogenerated with https://github.com/databricks/cli -->
+
 # <DABS> resources
 
 <DABS> allows you to specify information about the <Databricks> resources used by the bundle in the `resources` mapping in the bundle configuration. See [resources mapping](/dev-tools/bundles/settings.md#resources) and [resources key reference](/dev-tools/bundles/reference.md#resources).

From 57b8d336e03cb2b9aa0c353d9cce7d85d5c72c7f Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Wed, 5 Feb 2025 15:02:15 +0530
Subject: [PATCH 198/247] Add ability to record headers in acceptance tests
 (#2296)

## Changes
HTTP headers like the User-Agent are an important part of our internal
ETL pipelines. This PR adds the ability to validate the headers used in
an HTTP request as part of our acceptance tests.

## Tests
Modifying existing test.
---
 acceptance/acceptance_test.go                 |  3 +++
 acceptance/config_test.go                     |  3 +++
 .../workspace/jobs/create/out.requests.txt    |  2 +-
 acceptance/workspace/jobs/create/test.toml    | 17 ++++++++++++
 libs/testdiff/replacement.go                  | 18 +++++++++++++
 libs/testserver/server.go                     | 26 ++++++++++++++-----
 6 files changed, 61 insertions(+), 8 deletions(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 6e302fa96..f205217ff 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -156,6 +156,8 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	testdiff.PrepareReplacementsWorkspaceClient(t, &repls, workspaceClient)
 	testdiff.PrepareReplacementsUUID(t, &repls)
 	testdiff.PrepareReplacementsDevVersion(t, &repls)
+	testdiff.PrepareReplacementSdkVersion(t, &repls)
+	testdiff.PrepareReplacementsGoVersion(t, &repls)
 
 	testDirs := getTests(t)
 	require.NotEmpty(t, testDirs)
@@ -253,6 +255,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 	if len(config.Server) > 0 || config.RecordRequests {
 		server = testserver.New(t)
 		server.RecordRequests = config.RecordRequests
+		server.IncludeRequestHeaders = config.IncludeRequestHeaders
 
 		// If no custom server stubs are defined, add the default handlers.
 		if len(config.Server) == 0 {
diff --git a/acceptance/config_test.go b/acceptance/config_test.go
index 97be457e2..e24a683e7 100644
--- a/acceptance/config_test.go
+++ b/acceptance/config_test.go
@@ -47,6 +47,9 @@ type TestConfig struct {
 	// Record the requests made to the server and write them as output to
 	// out.requests.txt
 	RecordRequests bool
+
+	// List of request headers to include when recording requests.
+	IncludeRequestHeaders []string
 }
 
 type ServerStub struct {
diff --git a/acceptance/workspace/jobs/create/out.requests.txt b/acceptance/workspace/jobs/create/out.requests.txt
index b22876b70..4a85c4c43 100644
--- a/acceptance/workspace/jobs/create/out.requests.txt
+++ b/acceptance/workspace/jobs/create/out.requests.txt
@@ -1 +1 @@
-{"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}}
+{"headers":{"Authorization":"Bearer dapi1234","User-Agent":"cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}}
diff --git a/acceptance/workspace/jobs/create/test.toml b/acceptance/workspace/jobs/create/test.toml
index e69569c18..1fd9b3cec 100644
--- a/acceptance/workspace/jobs/create/test.toml
+++ b/acceptance/workspace/jobs/create/test.toml
@@ -1,5 +1,6 @@
 LocalOnly = true  # request recording currently does not work with cloud environment
 RecordRequests = true
+IncludeRequestHeaders = ["Authorization", "User-Agent"]
 
 [[Server]]
 Pattern = "POST /api/2.1/jobs/create"
@@ -8,3 +9,19 @@ Response.Body = '''
     "job_id": 1111
 }
 '''
+
+[[Repls]]
+Old = "(linux|darwin|windows)"
+New = "[OS]"
+
+[[Repls]]
+Old = " upstream/[A-Za-z0-9.-]+"
+New = ""
+
+[[Repls]]
+Old = " upstream-version/[A-Za-z0-9.-]+"
+New = ""
+
+[[Repls]]
+Old = " cicd/[A-Za-z0-9.-]+"
+New = ""
diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go
index 5bbba1be1..d4d5eb27b 100644
--- a/libs/testdiff/replacement.go
+++ b/libs/testdiff/replacement.go
@@ -12,6 +12,7 @@ import (
 	"github.com/databricks/cli/libs/iamutil"
 	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/service/iam"
+	"golang.org/x/mod/semver"
 )
 
 const (
@@ -208,3 +209,20 @@ func PrepareReplacementsDevVersion(t testutil.TestingT, r *ReplacementsContext)
 	t.Helper()
 	r.append(devVersionRegex, "[DEV_VERSION]")
 }
+
+func PrepareReplacementSdkVersion(t testutil.TestingT, r *ReplacementsContext) {
+	t.Helper()
+	r.Set(databricks.Version(), "[SDK_VERSION]")
+}
+
+func goVersion() string {
+	gv := runtime.Version()
+	ssv := strings.ReplaceAll(gv, "go", "v")
+	sv := semver.Canonical(ssv)
+	return strings.TrimPrefix(sv, "v")
+}
+
+func PrepareReplacementsGoVersion(t testutil.TestingT, r *ReplacementsContext) {
+	t.Helper()
+	r.Set(goVersion(), "[GO_VERSION]")
+}
diff --git a/libs/testserver/server.go b/libs/testserver/server.go
index a751531ed..5e3efe1c5 100644
--- a/libs/testserver/server.go
+++ b/libs/testserver/server.go
@@ -5,6 +5,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"slices"
 
 	"github.com/stretchr/testify/assert"
 
@@ -17,15 +18,17 @@ type Server struct {
 
 	t testutil.TestingT
 
-	RecordRequests bool
+	RecordRequests        bool
+	IncludeRequestHeaders []string
 
 	Requests []Request
 }
 
 type Request struct {
-	Method string `json:"method"`
-	Path   string `json:"path"`
-	Body   any    `json:"body"`
+	Headers map[string]string `json:"headers,omitempty"`
+	Method  string            `json:"method"`
+	Path    string            `json:"path"`
+	Body    any               `json:"body"`
 }
 
 func New(t testutil.TestingT) *Server {
@@ -50,10 +53,19 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) {
 			body, err := io.ReadAll(r.Body)
 			assert.NoError(s.t, err)
 
+			headers := make(map[string]string)
+			for k, v := range r.Header {
+				if len(v) == 0 || !slices.Contains(s.IncludeRequestHeaders, k) {
+					continue
+				}
+				headers[k] = v[0]
+			}
+
 			s.Requests = append(s.Requests, Request{
-				Method: r.Method,
-				Path:   r.URL.Path,
-				Body:   json.RawMessage(body),
+				Headers: headers,
+				Method:  r.Method,
+				Path:    r.URL.Path,
+				Body:    json.RawMessage(body),
 			})
 
 		}

From 5c90752797d4cd2e49bdda5a05fb35e9a59d5edc Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Wed, 5 Feb 2025 11:53:36 +0000
Subject: [PATCH 199/247] acc: Added acceptance test for CLI commands inside
 bundle with and without profile flag (#2270)

## Changes
This encodes existing behaviour in CLI as reported here: #1358
---
 .../auth/bundle_and_profile/.databrickscfg    |  5 +++
 .../auth/bundle_and_profile/databricks.yml    | 14 ++++++++
 acceptance/auth/bundle_and_profile/output.txt | 32 +++++++++++++++++++
 acceptance/auth/bundle_and_profile/script     | 30 +++++++++++++++++
 acceptance/auth/bundle_and_profile/test.toml  |  8 +++++
 5 files changed, 89 insertions(+)
 create mode 100644 acceptance/auth/bundle_and_profile/.databrickscfg
 create mode 100644 acceptance/auth/bundle_and_profile/databricks.yml
 create mode 100644 acceptance/auth/bundle_and_profile/output.txt
 create mode 100644 acceptance/auth/bundle_and_profile/script
 create mode 100644 acceptance/auth/bundle_and_profile/test.toml

diff --git a/acceptance/auth/bundle_and_profile/.databrickscfg b/acceptance/auth/bundle_and_profile/.databrickscfg
new file mode 100644
index 000000000..628505286
--- /dev/null
+++ b/acceptance/auth/bundle_and_profile/.databrickscfg
@@ -0,0 +1,5 @@
+[DEFAULT]
+host = $DATABRICKS_HOST
+
+[profile_name]
+host = https://test@non-existing-subdomain.databricks.com
diff --git a/acceptance/auth/bundle_and_profile/databricks.yml b/acceptance/auth/bundle_and_profile/databricks.yml
new file mode 100644
index 000000000..975661395
--- /dev/null
+++ b/acceptance/auth/bundle_and_profile/databricks.yml
@@ -0,0 +1,14 @@
+bundle:
+  name: test-auth
+
+workspace:
+  host: $DATABRICKS_HOST
+
+targets:
+  dev:
+    default: true
+    workspace:
+      host: $DATABRICKS_HOST
+  prod:
+    workspace:
+      host: https://bar.com
diff --git a/acceptance/auth/bundle_and_profile/output.txt b/acceptance/auth/bundle_and_profile/output.txt
new file mode 100644
index 000000000..022b3148d
--- /dev/null
+++ b/acceptance/auth/bundle_and_profile/output.txt
@@ -0,0 +1,32 @@
+
+=== Inside the bundle, no flags
+>>> errcode [CLI] current-user me
+"[USERNAME]"
+
+=== Inside the bundle, target flags
+>>> errcode [CLI] current-user me -t dev
+"[USERNAME]"
+
+=== Inside the bundle, target and matching profile
+>>> errcode [CLI] current-user me -t dev -p DEFAULT
+"[USERNAME]"
+
+=== Inside the bundle, profile flag not matching bundle host. Badness: should use profile from flag instead and not fail
+>>> errcode [CLI] current-user me -p profile_name
+Error: cannot resolve bundle auth configuration: config host mismatch: profile uses host https://non-existing-subdomain.databricks.com, but CLI configured to use [DATABRICKS_URL]
+
+Exit code: 1
+
+=== Inside the bundle, target and not matching profile
+>>> errcode [CLI] current-user me -t dev -p profile_name
+Error: cannot resolve bundle auth configuration: config host mismatch: profile uses host https://non-existing-subdomain.databricks.com, but CLI configured to use [DATABRICKS_URL]
+
+Exit code: 1
+
+=== Outside the bundle, no flags
+>>> errcode [CLI] current-user me
+"[USERNAME]"
+
+=== Outside the bundle, profile flag
+>>> errcode [CLI] current-user me -p profile_name
+"[USERNAME]"
diff --git a/acceptance/auth/bundle_and_profile/script b/acceptance/auth/bundle_and_profile/script
new file mode 100644
index 000000000..b37f5e01d
--- /dev/null
+++ b/acceptance/auth/bundle_and_profile/script
@@ -0,0 +1,30 @@
+# Replace placeholder with an actual host URL
+envsubst < databricks.yml > out.yml && mv out.yml databricks.yml
+envsubst < .databrickscfg > out && mv out .databrickscfg
+export DATABRICKS_CONFIG_FILE=.databrickscfg
+
+host=$DATABRICKS_HOST
+unset DATABRICKS_HOST
+
+title "Inside the bundle, no flags"
+trace errcode $CLI current-user me | jq .userName
+
+title "Inside the bundle, target flags"
+trace errcode $CLI current-user me -t dev | jq .userName
+
+title "Inside the bundle, target and matching profile"
+trace errcode $CLI current-user me -t dev -p DEFAULT | jq .userName
+
+title "Inside the bundle, profile flag not matching bundle host. Badness: should use profile from flag instead and not fail"
+trace errcode $CLI current-user me -p profile_name | jq .userName
+
+title "Inside the bundle, target and not matching profile"
+trace errcode $CLI current-user me -t dev -p profile_name
+
+cd ..
+export DATABRICKS_HOST=$host
+title "Outside the bundle, no flags"
+trace errcode $CLI current-user me | jq .userName
+
+title "Outside the bundle, profile flag"
+trace errcode $CLI current-user me -p profile_name | jq .userName
diff --git a/acceptance/auth/bundle_and_profile/test.toml b/acceptance/auth/bundle_and_profile/test.toml
new file mode 100644
index 000000000..b20190ca5
--- /dev/null
+++ b/acceptance/auth/bundle_and_profile/test.toml
@@ -0,0 +1,8 @@
+Badness = "When -p flag is used inside the bundle folder for any CLI commands, CLI use bundle host anyway instead of profile one"
+
+# Some of the clouds have DATABRICKS_HOST variable setup without https:// prefix
+# In the result, output is replaced with DATABRICKS_URL variable instead of DATABRICKS_HOST
+# This is a workaround to replace DATABRICKS_URL with DATABRICKS_HOST
+[[Repls]]
+Old='DATABRICKS_HOST'
+New='DATABRICKS_URL'

From 27caf413f2d601de8d81d6cc3fb2614ff1bf354c Mon Sep 17 00:00:00 2001
From: Marcin Wojtyczka <marcin.wojtyczka@databricks.com>
Date: Wed, 5 Feb 2025 14:24:15 +0100
Subject: [PATCH 200/247] Add support for extras to the labs CLI (#2288)

## Changes

Added support for extras / optional Python dependencies in the labs CLI.

Added new `extras` field under install.

Example:

```yaml
install:
  script: install.py
  extras: cli
```

Resolves: #2257

## Tests

Manual test
---
 cmd/labs/project/installer.go                                | 5 +++++
 cmd/labs/project/schema.json                                 | 5 +++++
 .../.databricks/labs/blueprint/lib/labs.yml                  | 1 +
 3 files changed, 11 insertions(+)

diff --git a/cmd/labs/project/installer.go b/cmd/labs/project/installer.go
index 05f7d68aa..2e42ce43d 100644
--- a/cmd/labs/project/installer.go
+++ b/cmd/labs/project/installer.go
@@ -32,6 +32,7 @@ type hook struct {
 	RequireDatabricksConnect bool    `yaml:"require_databricks_connect,omitempty"`
 	MinRuntimeVersion        string  `yaml:"min_runtime_version,omitempty"`
 	WarehouseTypes           whTypes `yaml:"warehouse_types,omitempty"`
+	Extras                   string  `yaml:"extras,omitempty"`
 }
 
 func (h *hook) RequireRunningCluster() bool {
@@ -258,6 +259,10 @@ func (i *installer) setupPythonVirtualEnvironment(ctx context.Context, w *databr
 		}
 	}
 	feedback <- "Installing Python library dependencies"
+	if i.Installer.Extras != "" {
+		// install main and optional dependencies
+		return i.installPythonDependencies(ctx, fmt.Sprintf(".[%s]", i.Installer.Extras))
+	}
 	return i.installPythonDependencies(ctx, ".")
 }
 
diff --git a/cmd/labs/project/schema.json b/cmd/labs/project/schema.json
index a779b15e4..7aa65813c 100644
--- a/cmd/labs/project/schema.json
+++ b/cmd/labs/project/schema.json
@@ -42,6 +42,11 @@
                 },
                 "warehouse_types": {
                     "enum": [ "PRO", "CLASSIC", "TYPE_UNSPECIFIED" ]
+                },
+                "extras": {
+                    "type": "string",
+                    "pattern": "^([^,]+)(,([^,]+))*$",
+                    "default": ""
                 }
             }
         },
diff --git a/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml b/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml
index 0ac4bf826..b8a0e695e 100644
--- a/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml
+++ b/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml
@@ -8,6 +8,7 @@ install:
   warehouse_types:
     - PRO
   script: install.py
+  extras: ""
 entrypoint: main.py
 min_python: 3.9
 commands:

From e0903fbd3712c252874f193076a88d6a83cd966e Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 5 Feb 2025 15:58:29 +0100
Subject: [PATCH 201/247] Include 'go mod tidy' into 'make' and 'make tidy'
 (#2298)

Apparently, it's not part of golangci-lint, so you can send PRs that
fail this check on CI.
---
 Makefile | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index e18727934..7da7e4789 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-default: vendor fmt lint
+default: vendor fmt lint tidy
 
 PACKAGES=./acceptance/... ./libs/... ./internal/... ./cmd/... ./bundle/... .
 
@@ -9,6 +9,10 @@ GOTESTSUM_CMD ?= gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped
 lint:
 	golangci-lint run --fix
 
+tidy:
+	# not part of golangci-lint, apparently
+	go mod tidy
+
 lintcheck:
 	golangci-lint run ./...
 
@@ -59,4 +63,4 @@ integration: vendor
 integration-short: vendor
 	$(INTEGRATION) -short
 
-.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover docs
+.PHONY: lint tidy lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover docs

From 27eb0c40725aa978191ce3fe43eb07993fd6cfe4 Mon Sep 17 00:00:00 2001
From: Ilya Kuznetsov <ilya.kuznetsov@databricks.com>
Date: Thu, 6 Feb 2025 20:27:55 +0100
Subject: [PATCH 202/247] Allow 'any' examples in JSON schema (#2289)

## Changes

1. Allow `any` examples in json-schema type since we have many of them
in open api spec
2. Fix issue with missing overrides annotations when re-generating the
schema

## Tests
<!-- How is this tested? -->
---
 bundle/docsgen/main.go                        |   2 +-
 bundle/docsgen/nodes.go                       |   4 +-
 bundle/docsgen/refs.go                        |  12 +-
 .../schema/annotations_openapi_overrides.yml  | 161 +++++++++---------
 bundle/internal/schema/parser.go              |  10 +-
 libs/jsonschema/schema.go                     |   2 +-
 6 files changed, 101 insertions(+), 90 deletions(-)

diff --git a/bundle/docsgen/main.go b/bundle/docsgen/main.go
index ad737feea..84bf4779f 100644
--- a/bundle/docsgen/main.go
+++ b/bundle/docsgen/main.go
@@ -130,6 +130,6 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) {
 		s.MarkdownDescription = a.MarkdownDescription
 	}
 	if a.MarkdownExamples != "" {
-		s.Examples = []any{a.MarkdownExamples}
+		s.Examples = []string{a.MarkdownExamples}
 	}
 }
diff --git a/bundle/docsgen/nodes.go b/bundle/docsgen/nodes.go
index 68ed86450..61d2c21cc 100644
--- a/bundle/docsgen/nodes.go
+++ b/bundle/docsgen/nodes.go
@@ -220,9 +220,9 @@ func isCycleField(field string) bool {
 }
 
 func getExample(v *jsonschema.Schema) string {
-	examples := v.Examples
+	examples := getExamples(v.Examples)
 	if len(examples) == 0 {
 		return ""
 	}
-	return examples[0].(string)
+	return examples[0]
 }
diff --git a/bundle/docsgen/refs.go b/bundle/docsgen/refs.go
index ca45e6ab2..7a4451129 100644
--- a/bundle/docsgen/refs.go
+++ b/bundle/docsgen/refs.go
@@ -58,7 +58,7 @@ func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *j
 	node := s
 	description := s.Description
 	markdownDescription := s.MarkdownDescription
-	examples := s.Examples
+	examples := getExamples(s.Examples)
 
 	for node.Reference != nil {
 		ref := getRefType(node)
@@ -75,7 +75,7 @@ func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *j
 			markdownDescription = newNode.MarkdownDescription
 		}
 		if len(examples) == 0 {
-			examples = newNode.Examples
+			examples = getExamples(newNode.Examples)
 		}
 
 		node = newNode
@@ -89,6 +89,14 @@ func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *j
 	return &newNode
 }
 
+func getExamples(examples any) []string {
+	typedExamples, ok := examples.([]string)
+	if !ok {
+		return []string{}
+	}
+	return typedExamples
+}
+
 func getRefType(node *jsonschema.Schema) string {
 	if node.Reference == nil {
 		return ""
diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml
index 912a4fda0..585886313 100644
--- a/bundle/internal/schema/annotations_openapi_overrides.yml
+++ b/bundle/internal/schema/annotations_openapi_overrides.yml
@@ -60,7 +60,6 @@ github.com/databricks/cli/bundle/config/resources.Cluster:
   "_":
     "markdown_description": |-
       The cluster resource defines an [all-purpose cluster](/api/workspace/clusters/create).
-
     "markdown_examples": |-
       The following example creates a cluster named `my_cluster` and sets that as the cluster to use to run the notebook in `my_job`:
 
@@ -123,7 +122,6 @@ github.com/databricks/cli/bundle/config/resources.Dashboard:
       If you use the UI to modify the dashboard, modifications made through the UI are not applied to the dashboard JSON file in the local bundle unless you explicitly update it using `bundle generate`. You can use the `--watch` option to continuously poll and retrieve changes to the dashboard. See [_](/dev-tools/cli/bundle-commands.md#generate).
 
       In addition, if you attempt to deploy a bundle that contains a dashboard JSON file that is different than the one in the remote workspace, an error will occur. To force the deploy and overwrite the dashboard in the remote workspace with the local one, use the `--force` option. See [_](/dev-tools/cli/bundle-commands.md#deploy).
-
   "embed_credentials":
     "description": |-
       PLACEHOLDER
@@ -356,7 +354,6 @@ github.com/databricks/cli/bundle/config/resources.Volume:
       - A volume cannot be referenced in the `artifact_path` for the bundle until it exists in the workspace. Hence, if you want to use <DABS> to create the volume, you must first define the volume in the bundle, deploy it to create the volume, then reference it in the `artifact_path` in subsequent deployments.
 
       - Volumes in the bundle are not prepended with the `dev_${workspace.current_user.short_name}` prefix when the deployment target has `mode: development` configured. However, you can manually configure this prefix. See [_](/dev-tools/bundles/deployment-modes.md#custom-presets).
-
     "markdown_examples": |-
       The following example creates a <UC> volume with the key `my_volume`:
 
@@ -376,6 +373,42 @@ github.com/databricks/cli/bundle/config/resources.Volume:
   "volume_type":
     "description": |-
       PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppDeployment:
+  "create_time":
+    "description": |-
+      PLACEHOLDER
+  "creator":
+    "description": |-
+      PLACEHOLDER
+  "deployment_artifacts":
+    "description": |-
+      PLACEHOLDER
+  "deployment_id":
+    "description": |-
+      PLACEHOLDER
+  "mode":
+    "description": |-
+      PLACEHOLDER
+  "source_code_path":
+    "description": |-
+      PLACEHOLDER
+  "status":
+    "description": |-
+      PLACEHOLDER
+  "update_time":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts:
+  "source_code_path":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus:
+  "message":
+    "description": |-
+      PLACEHOLDER
+  "state":
+    "description": |-
+      PLACEHOLDER
 github.com/databricks/databricks-sdk-go/service/apps.AppResource:
   "job":
     "description": |-
@@ -389,6 +422,49 @@ github.com/databricks/databricks-sdk-go/service/apps.AppResource:
   "sql_warehouse":
     "description": |-
       PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob:
+  "id":
+    "description": |-
+      PLACEHOLDER
+  "permission":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret:
+  "key":
+    "description": |-
+      PLACEHOLDER
+  "permission":
+    "description": |-
+      PLACEHOLDER
+  "scope":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint:
+  "name":
+    "description": |-
+      PLACEHOLDER
+  "permission":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse:
+  "id":
+    "description": |-
+      PLACEHOLDER
+  "permission":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus:
+  "message":
+    "description": |-
+      PLACEHOLDER
+  "state":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus:
+  "message":
+    "description": |-
+      PLACEHOLDER
+  "state": {}
 github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes:
   "availability":
     "description": |-
@@ -473,85 +549,6 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger:
   "manual":
     "description": |-
       PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppDeployment:
-  "create_time":
-    "description": |-
-      PLACEHOLDER
-  "creator":
-    "description": |-
-      PLACEHOLDER
-  "deployment_artifacts":
-    "description": |-
-      PLACEHOLDER
-  "deployment_id":
-    "description": |-
-      PLACEHOLDER
-  "mode":
-    "description": |-
-      PLACEHOLDER
-  "source_code_path":
-    "description": |-
-      PLACEHOLDER
-  "status":
-    "description": |-
-      PLACEHOLDER
-  "update_time":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts:
-  "source_code_path":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus:
-  "message":
-    "description": |-
-      PLACEHOLDER
-  "state":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob:
-  "id":
-    "description": |-
-      PLACEHOLDER
-  "permission":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret:
-  "key":
-    "description": |-
-      PLACEHOLDER
-  "permission":
-    "description": |-
-      PLACEHOLDER
-  "scope":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint:
-  "name":
-    "description": |-
-      PLACEHOLDER
-  "permission":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse:
-  "id":
-    "description": |-
-      PLACEHOLDER
-  "permission":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus:
-  "message":
-    "description": |-
-      PLACEHOLDER
-  "state":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus:
-  "message":
-    "description": |-
-      PLACEHOLDER
-  "state":
 github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput:
   "entity_version":
     "description": |-
diff --git a/bundle/internal/schema/parser.go b/bundle/internal/schema/parser.go
index 50e69e7c8..ca8c27d4c 100644
--- a/bundle/internal/schema/parser.go
+++ b/bundle/internal/schema/parser.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -9,8 +10,9 @@ import (
 	"strings"
 
 	"github.com/databricks/cli/bundle/internal/annotation"
+	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/databricks/cli/libs/dyn/yamlloader"
 	"github.com/databricks/cli/libs/jsonschema"
-	"gopkg.in/yaml.v3"
 )
 
 type Components struct {
@@ -122,7 +124,11 @@ func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overrid
 	if err != nil {
 		return err
 	}
-	err = yaml.Unmarshal(b, &overrides)
+	overridesDyn, err := yamlloader.LoadYAML(overridesPath, bytes.NewBuffer(b))
+	if err != nil {
+		return err
+	}
+	err = convert.ToTyped(&overrides, overridesDyn)
 	if err != nil {
 		return err
 	}
diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go
index 5028bb0d7..85f6a0328 100644
--- a/libs/jsonschema/schema.go
+++ b/libs/jsonschema/schema.go
@@ -79,7 +79,7 @@ type Schema struct {
 
 	// Examples of the value for properties in the schema.
 	// https://json-schema.org/understanding-json-schema/reference/annotations
-	Examples []any `json:"examples,omitempty"`
+	Examples any `json:"examples,omitempty"`
 }
 
 // Default value defined in a JSON Schema, represented as a string.

From 75127fe42eb5fb2221b49c39977de32e3441ae0d Mon Sep 17 00:00:00 2001
From: Gleb Kanterov <kanterov@users.noreply.github.com>
Date: Fri, 7 Feb 2025 11:26:20 +0100
Subject: [PATCH 203/247] Extend testserver for deployment (#2299)

## Changes
Extend testserver for bundle deployment:

- Allocate a new workspace per test case to isolate test cases from each
other
- Support jobs get/list/create
- Support creation and listing of workspace files

## Tests
Using existing acceptance tests
---
 acceptance/acceptance_test.go                 |  28 ++-
 acceptance/bundle/scripts/output.txt          |  14 +-
 acceptance/cmd_server_test.go                 |   2 +-
 acceptance/server_test.go                     | 107 +++++++++--
 .../workspace/jobs/create/out.requests.txt    |   2 +-
 libs/testserver/fake_workspace.go             | 169 ++++++++++++++++++
 libs/testserver/server.go                     |  50 +++++-
 7 files changed, 332 insertions(+), 40 deletions(-)
 create mode 100644 libs/testserver/fake_workspace.go

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index f205217ff..4c4404d55 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -19,6 +19,8 @@ import (
 	"time"
 	"unicode/utf8"
 
+	"github.com/google/uuid"
+
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/env"
 	"github.com/databricks/cli/libs/testdiff"
@@ -123,7 +125,6 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 		AddHandlers(defaultServer)
 		// Redirect API access to local server:
 		t.Setenv("DATABRICKS_HOST", defaultServer.URL)
-		t.Setenv("DATABRICKS_TOKEN", "dapi1234")
 
 		homeDir := t.TempDir()
 		// Do not read user's ~/.databrickscfg
@@ -146,7 +147,15 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	// do it last so that full paths match first:
 	repls.SetPath(buildDir, "[BUILD_DIR]")
 
-	workspaceClient, err := databricks.NewWorkspaceClient()
+	var config databricks.Config
+	if cloudEnv == "" {
+		// use fake token for local tests
+		config = databricks.Config{Token: "dbapi1234"}
+	} else {
+		// non-local tests rely on environment variables
+		config = databricks.Config{}
+	}
+	workspaceClient, err := databricks.NewWorkspaceClient(&config)
 	require.NoError(t, err)
 
 	user, err := workspaceClient.CurrentUser.Me(ctx)
@@ -264,7 +273,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 
 		for _, stub := range config.Server {
 			require.NotEmpty(t, stub.Pattern)
-			server.Handle(stub.Pattern, func(req *http.Request) (any, int) {
+			server.Handle(stub.Pattern, func(fakeWorkspace *testserver.FakeWorkspace, req *http.Request) (any, int) {
 				statusCode := http.StatusOK
 				if stub.Response.StatusCode != 0 {
 					statusCode = stub.Response.StatusCode
@@ -285,6 +294,15 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir)
 	}
 
+	// Each local test should use a new token that will result into a new fake workspace,
+	// so that test don't interfere with each other.
+	if cloudEnv == "" {
+		tokenSuffix := strings.ReplaceAll(uuid.NewString(), "-", "")
+		token := "dbapi" + tokenSuffix
+		cmd.Env = append(cmd.Env, "DATABRICKS_TOKEN="+token)
+		repls.Set(token, "[DATABRICKS_TOKEN]")
+	}
+
 	// Write combined output to a file
 	out, err := os.Create(filepath.Join(tmpDir, "output.txt"))
 	require.NoError(t, err)
@@ -303,8 +321,8 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 			reqJson, err := json.Marshal(req)
 			require.NoError(t, err)
 
-			line := fmt.Sprintf("%s\n", reqJson)
-			_, err = f.WriteString(line)
+			reqJsonWithRepls := repls.Replace(string(reqJson))
+			_, err = f.WriteString(reqJsonWithRepls + "\n")
 			require.NoError(t, err)
 		}
 
diff --git a/acceptance/bundle/scripts/output.txt b/acceptance/bundle/scripts/output.txt
index 2deedb0e7..68afb2fec 100644
--- a/acceptance/bundle/scripts/output.txt
+++ b/acceptance/bundle/scripts/output.txt
@@ -42,11 +42,9 @@ from myscript.py 0 postbuild: hello stderr!
 Executing 'predeploy' script
 from myscript.py 0 predeploy: hello stdout!
 from myscript.py 0 predeploy: hello stderr!
-Error: unable to deploy to /Workspace/Users/[USERNAME]/.bundle/scripts/default/state as [USERNAME].
-Please make sure the current user or one of their groups is listed under the permissions of this bundle.
-For assistance, contact the owners of this project.
-They may need to redeploy the bundle to apply the new permissions.
-Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.
-
-
-Exit code: 1
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/scripts/default/files...
+Deploying resources...
+Deployment complete!
+Executing 'postdeploy' script
+from myscript.py 0 postdeploy: hello stdout!
+from myscript.py 0 postdeploy: hello stderr!
diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go
index 04d56c7d4..0166dfe32 100644
--- a/acceptance/cmd_server_test.go
+++ b/acceptance/cmd_server_test.go
@@ -15,7 +15,7 @@ import (
 func StartCmdServer(t *testing.T) *testserver.Server {
 	server := testserver.New(t)
 
-	server.Handle("/", func(r *http.Request) (any, int) {
+	server.Handle("/", func(w *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		q := r.URL.Query()
 		args := strings.Split(q.Get("args"), " ")
 
diff --git a/acceptance/server_test.go b/acceptance/server_test.go
index a7695b21e..d21ab66e8 100644
--- a/acceptance/server_test.go
+++ b/acceptance/server_test.go
@@ -1,17 +1,23 @@
 package acceptance_test
 
 import (
+	"bytes"
+	"encoding/json"
+	"fmt"
 	"net/http"
 
-	"github.com/databricks/cli/libs/testserver"
 	"github.com/databricks/databricks-sdk-go/service/catalog"
-	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/iam"
+
+	"github.com/databricks/databricks-sdk-go/service/compute"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+
+	"github.com/databricks/cli/libs/testserver"
 	"github.com/databricks/databricks-sdk-go/service/workspace"
 )
 
 func AddHandlers(server *testserver.Server) {
-	server.Handle("GET /api/2.0/policies/clusters/list", func(r *http.Request) (any, int) {
+	server.Handle("GET /api/2.0/policies/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		return compute.ListPoliciesResponse{
 			Policies: []compute.Policy{
 				{
@@ -26,7 +32,7 @@ func AddHandlers(server *testserver.Server) {
 		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.0/instance-pools/list", func(r *http.Request) (any, int) {
+	server.Handle("GET /api/2.0/instance-pools/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		return compute.ListInstancePools{
 			InstancePools: []compute.InstancePoolAndStats{
 				{
@@ -37,7 +43,7 @@ func AddHandlers(server *testserver.Server) {
 		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.1/clusters/list", func(r *http.Request) (any, int) {
+	server.Handle("GET /api/2.1/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		return compute.ListClustersResponse{
 			Clusters: []compute.ClusterDetails{
 				{
@@ -52,31 +58,74 @@ func AddHandlers(server *testserver.Server) {
 		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, int) {
+	server.Handle("GET /api/2.0/preview/scim/v2/Me", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		return iam.User{
 			Id:       "1000012345",
 			UserName: "tester@databricks.com",
 		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.0/workspace/get-status", func(r *http.Request) (any, int) {
-		return workspace.ObjectInfo{
-			ObjectId:   1001,
-			ObjectType: "DIRECTORY",
-			Path:       "",
-			ResourceId: "1001",
-		}, http.StatusOK
+	server.Handle("GET /api/2.0/workspace/get-status", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		path := r.URL.Query().Get("path")
+
+		return fakeWorkspace.WorkspaceGetStatus(path)
 	})
 
-	server.Handle("GET /api/2.1/unity-catalog/current-metastore-assignment", func(r *http.Request) (any, int) {
+	server.Handle("POST /api/2.0/workspace/mkdirs", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		request := workspace.Mkdirs{}
+		decoder := json.NewDecoder(r.Body)
+
+		err := decoder.Decode(&request)
+		if err != nil {
+			return internalError(err)
+		}
+
+		return fakeWorkspace.WorkspaceMkdirs(request)
+	})
+
+	server.Handle("GET /api/2.0/workspace/export", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		path := r.URL.Query().Get("path")
+
+		return fakeWorkspace.WorkspaceExport(path)
+	})
+
+	server.Handle("POST /api/2.0/workspace/delete", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		path := r.URL.Query().Get("path")
+		recursiveStr := r.URL.Query().Get("recursive")
+		var recursive bool
+
+		if recursiveStr == "true" {
+			recursive = true
+		} else {
+			recursive = false
+		}
+
+		return fakeWorkspace.WorkspaceDelete(path, recursive)
+	})
+
+	server.Handle("POST /api/2.0/workspace-files/import-file/{path}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		path := r.PathValue("path")
+
+		body := new(bytes.Buffer)
+		_, err := body.ReadFrom(r.Body)
+		if err != nil {
+			return internalError(err)
+		}
+
+		return fakeWorkspace.WorkspaceFilesImportFile(path, body.Bytes())
+	})
+
+	server.Handle("GET /api/2.1/unity-catalog/current-metastore-assignment", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		return catalog.MetastoreAssignment{
 			DefaultCatalogName: "main",
 		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.0/permissions/directories/1001", func(r *http.Request) (any, int) {
+	server.Handle("GET /api/2.0/permissions/directories/{objectId}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		objectId := r.PathValue("objectId")
+
 		return workspace.WorkspaceObjectPermissions{
-			ObjectId:   "1001",
+			ObjectId:   objectId,
 			ObjectType: "DIRECTORY",
 			AccessControlList: []workspace.WorkspaceObjectAccessControlResponse{
 				{
@@ -91,7 +140,29 @@ func AddHandlers(server *testserver.Server) {
 		}, http.StatusOK
 	})
 
-	server.Handle("POST /api/2.0/workspace/mkdirs", func(r *http.Request) (any, int) {
-		return "{}", http.StatusOK
+	server.Handle("POST /api/2.1/jobs/create", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		request := jobs.CreateJob{}
+		decoder := json.NewDecoder(r.Body)
+
+		err := decoder.Decode(&request)
+		if err != nil {
+			return internalError(err)
+		}
+
+		return fakeWorkspace.JobsCreate(request)
+	})
+
+	server.Handle("GET /api/2.1/jobs/get", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		jobId := r.URL.Query().Get("job_id")
+
+		return fakeWorkspace.JobsGet(jobId)
+	})
+
+	server.Handle("GET /api/2.1/jobs/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		return fakeWorkspace.JobsList()
 	})
 }
+
+func internalError(err error) (any, int) {
+	return fmt.Errorf("internal error: %w", err), http.StatusInternalServerError
+}
diff --git a/acceptance/workspace/jobs/create/out.requests.txt b/acceptance/workspace/jobs/create/out.requests.txt
index 4a85c4c43..60977e3e3 100644
--- a/acceptance/workspace/jobs/create/out.requests.txt
+++ b/acceptance/workspace/jobs/create/out.requests.txt
@@ -1 +1 @@
-{"headers":{"Authorization":"Bearer dapi1234","User-Agent":"cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}}
+{"headers":{"Authorization":"Bearer [DATABRICKS_TOKEN]","User-Agent":"cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}}
diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go
new file mode 100644
index 000000000..c3e4f9a71
--- /dev/null
+++ b/libs/testserver/fake_workspace.go
@@ -0,0 +1,169 @@
+package testserver
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/databricks/databricks-sdk-go/service/workspace"
+)
+
+// FakeWorkspace holds a state of a workspace for acceptance tests.
+type FakeWorkspace struct {
+	directories map[string]bool
+	files       map[string][]byte
+	// normally, ids are not sequential, but we make them sequential for deterministic diff
+	nextJobId int64
+	jobs      map[int64]jobs.Job
+}
+
+func NewFakeWorkspace() *FakeWorkspace {
+	return &FakeWorkspace{
+		directories: map[string]bool{
+			"/Workspace": true,
+		},
+		files:     map[string][]byte{},
+		jobs:      map[int64]jobs.Job{},
+		nextJobId: 1,
+	}
+}
+
+func (s *FakeWorkspace) WorkspaceGetStatus(path string) (workspace.ObjectInfo, int) {
+	if s.directories[path] {
+		return workspace.ObjectInfo{
+			ObjectType: "DIRECTORY",
+			Path:       path,
+		}, http.StatusOK
+	} else if _, ok := s.files[path]; ok {
+		return workspace.ObjectInfo{
+			ObjectType: "FILE",
+			Path:       path,
+			Language:   "SCALA",
+		}, http.StatusOK
+	} else {
+		return workspace.ObjectInfo{}, http.StatusNotFound
+	}
+}
+
+func (s *FakeWorkspace) WorkspaceMkdirs(request workspace.Mkdirs) (string, int) {
+	s.directories[request.Path] = true
+
+	return "{}", http.StatusOK
+}
+
+func (s *FakeWorkspace) WorkspaceExport(path string) ([]byte, int) {
+	file := s.files[path]
+
+	if file == nil {
+		return nil, http.StatusNotFound
+	}
+
+	return file, http.StatusOK
+}
+
+func (s *FakeWorkspace) WorkspaceDelete(path string, recursive bool) (string, int) {
+	if !recursive {
+		s.files[path] = nil
+	} else {
+		for key := range s.files {
+			if strings.HasPrefix(key, path) {
+				s.files[key] = nil
+			}
+		}
+	}
+
+	return "{}", http.StatusOK
+}
+
+func (s *FakeWorkspace) WorkspaceFilesImportFile(path string, body []byte) (any, int) {
+	if !strings.HasPrefix(path, "/") {
+		path = "/" + path
+	}
+
+	s.files[path] = body
+
+	return "{}", http.StatusOK
+}
+
+func (s *FakeWorkspace) JobsCreate(request jobs.CreateJob) (any, int) {
+	jobId := s.nextJobId
+	s.nextJobId++
+
+	jobSettings := jobs.JobSettings{}
+	err := jsonConvert(request, &jobSettings)
+	if err != nil {
+		return internalError(err)
+	}
+
+	s.jobs[jobId] = jobs.Job{
+		JobId:    jobId,
+		Settings: &jobSettings,
+	}
+
+	return jobs.CreateResponse{JobId: jobId}, http.StatusOK
+}
+
+func (s *FakeWorkspace) JobsGet(jobId string) (any, int) {
+	id := jobId
+
+	jobIdInt, err := strconv.ParseInt(id, 10, 64)
+	if err != nil {
+		return internalError(fmt.Errorf("failed to parse job id: %s", err))
+	}
+
+	job, ok := s.jobs[jobIdInt]
+	if !ok {
+		return jobs.Job{}, http.StatusNotFound
+	}
+
+	return job, http.StatusOK
+}
+
+func (s *FakeWorkspace) JobsList() (any, int) {
+	list := make([]jobs.BaseJob, 0, len(s.jobs))
+	for _, job := range s.jobs {
+		baseJob := jobs.BaseJob{}
+		err := jsonConvert(job, &baseJob)
+		if err != nil {
+			return internalError(fmt.Errorf("failed to convert job to base job: %w", err))
+		}
+
+		list = append(list, baseJob)
+	}
+
+	// sort to have less non-determinism in tests
+	sort.Slice(list, func(i, j int) bool {
+		return list[i].JobId < list[j].JobId
+	})
+
+	return jobs.ListJobsResponse{
+		Jobs: list,
+	}, http.StatusOK
+}
+
+// jsonConvert saves input to a value pointed by output
+func jsonConvert(input, output any) error {
+	writer := new(bytes.Buffer)
+	encoder := json.NewEncoder(writer)
+	err := encoder.Encode(input)
+	if err != nil {
+		return fmt.Errorf("failed to encode: %w", err)
+	}
+
+	decoder := json.NewDecoder(writer)
+	err = decoder.Decode(output)
+	if err != nil {
+		return fmt.Errorf("failed to decode: %w", err)
+	}
+
+	return nil
+}
+
+func internalError(err error) (string, int) {
+	return fmt.Sprintf("internal error: %s", err), http.StatusInternalServerError
+}
diff --git a/libs/testserver/server.go b/libs/testserver/server.go
index 5e3efe1c5..ffb83a49c 100644
--- a/libs/testserver/server.go
+++ b/libs/testserver/server.go
@@ -6,6 +6,8 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"slices"
+	"strings"
+	"sync"
 
 	"github.com/stretchr/testify/assert"
 
@@ -18,6 +20,9 @@ type Server struct {
 
 	t testutil.TestingT
 
+	fakeWorkspaces map[string]*FakeWorkspace
+	mu             *sync.Mutex
+
 	RecordRequests        bool
 	IncludeRequestHeaders []string
 
@@ -37,17 +42,36 @@ func New(t testutil.TestingT) *Server {
 	t.Cleanup(server.Close)
 
 	return &Server{
-		Server: server,
-		Mux:    mux,
-		t:      t,
+		Server:         server,
+		Mux:            mux,
+		t:              t,
+		mu:             &sync.Mutex{},
+		fakeWorkspaces: map[string]*FakeWorkspace{},
 	}
 }
 
-type HandlerFunc func(req *http.Request) (resp any, statusCode int)
+type HandlerFunc func(fakeWorkspace *FakeWorkspace, req *http.Request) (resp any, statusCode int)
 
 func (s *Server) Handle(pattern string, handler HandlerFunc) {
 	s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) {
-		resp, statusCode := handler(r)
+		// For simplicity we process requests sequentially. It's fast enough because
+		// we don't do any IO except reading and writing request/response bodies.
+		s.mu.Lock()
+		defer s.mu.Unlock()
+
+		// Each test uses unique DATABRICKS_TOKEN, we simulate each token having
+		// it's own fake fakeWorkspace to avoid interference between tests.
+		var fakeWorkspace *FakeWorkspace = nil
+		token := getToken(r)
+		if token != "" {
+			if _, ok := s.fakeWorkspaces[token]; !ok {
+				s.fakeWorkspaces[token] = NewFakeWorkspace()
+			}
+
+			fakeWorkspace = s.fakeWorkspaces[token]
+		}
+
+		resp, statusCode := handler(fakeWorkspace, r)
 
 		if s.RecordRequests {
 			body, err := io.ReadAll(r.Body)
@@ -75,9 +99,10 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) {
 
 		var respBytes []byte
 		var err error
-		respString, ok := resp.(string)
-		if ok {
+		if respString, ok := resp.(string); ok {
 			respBytes = []byte(respString)
+		} else if respBytes0, ok := resp.([]byte); ok {
+			respBytes = respBytes0
 		} else {
 			respBytes, err = json.MarshalIndent(resp, "", "    ")
 			if err != nil {
@@ -92,3 +117,14 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) {
 		}
 	})
 }
+
+func getToken(r *http.Request) string {
+	header := r.Header.Get("Authorization")
+	prefix := "Bearer "
+
+	if !strings.HasPrefix(header, prefix) {
+		return ""
+	}
+
+	return header[len(prefix):]
+}

From 54e16d5f62ccd169c5f64319f3b174f6dc58e326 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 7 Feb 2025 12:29:40 +0100
Subject: [PATCH 204/247] Always print warnings and errors; clean up format
 (#2213)

## Changes
- Print warnings and errors by default.
- Fix ErrAlreadyPrinted not to be logged at Error level.
- Format log messages as "Warn: message" instead of "WARN" to make it
more readable and in-line with the rest of the output.
- Only print attributes (pid, mutator, etc) and time when the overall
level is debug (so --debug output has not changed much).

## Tests
- Existing acceptance tests show how warning messages appear in various
test case.
- Added new test for `--debug` output.
- Add sort_lines.py helper to avoid dependency on 'sort' which is
locale-sensitive.
---
 acceptance/bin/sort_lines.py                  | 10 ++
 acceptance/bundle/debug/databricks.yml        |  2 +
 .../bundle/debug/out.stderr.parallel.txt      | 15 +++
 acceptance/bundle/debug/out.stderr.txt        | 92 +++++++++++++++++++
 acceptance/bundle/debug/output.txt            |  7 ++
 acceptance/bundle/debug/script                |  4 +
 acceptance/bundle/debug/test.toml             | 18 ++++
 acceptance/bundle/git-permerror/output.txt    |  9 ++
 acceptance/bundle/git-permerror/test.toml     |  2 +-
 bundle/artifacts/upload.go                    |  2 +-
 bundle/mutator.go                             |  2 +-
 bundle/mutator_read_only.go                   |  2 +-
 cmd/root/root.go                              | 11 ++-
 libs/flags/log_level_flag.go                  |  2 +-
 libs/flags/log_level_flag_test.go             |  4 +-
 libs/log/handler/friendly.go                  | 60 ++++++------
 16 files changed, 206 insertions(+), 36 deletions(-)
 create mode 100755 acceptance/bin/sort_lines.py
 create mode 100644 acceptance/bundle/debug/databricks.yml
 create mode 100644 acceptance/bundle/debug/out.stderr.parallel.txt
 create mode 100644 acceptance/bundle/debug/out.stderr.txt
 create mode 100644 acceptance/bundle/debug/output.txt
 create mode 100644 acceptance/bundle/debug/script
 create mode 100644 acceptance/bundle/debug/test.toml

diff --git a/acceptance/bin/sort_lines.py b/acceptance/bin/sort_lines.py
new file mode 100755
index 000000000..9ac87feee
--- /dev/null
+++ b/acceptance/bin/sort_lines.py
@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+"""
+Helper to sort lines in text file. Similar to 'sort' but no dependence on locale or presence of 'sort' in PATH.
+"""
+
+import sys
+
+lines = sys.stdin.readlines()
+lines.sort()
+sys.stdout.write("".join(lines))
diff --git a/acceptance/bundle/debug/databricks.yml b/acceptance/bundle/debug/databricks.yml
new file mode 100644
index 000000000..2c9dd3c90
--- /dev/null
+++ b/acceptance/bundle/debug/databricks.yml
@@ -0,0 +1,2 @@
+bundle:
+  name: debug
diff --git a/acceptance/bundle/debug/out.stderr.parallel.txt b/acceptance/bundle/debug/out.stderr.parallel.txt
new file mode 100644
index 000000000..7dd770068
--- /dev/null
+++ b/acceptance/bundle/debug/out.stderr.parallel.txt
@@ -0,0 +1,15 @@
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly)
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:SingleNodeCluster
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:artifact_paths
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:job_cluster_key_defined
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:job_task_cluster_spec
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:folder_permissions
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:validate_sync_patterns
+10:07:59 Debug: Path /Workspace/Users/[USERNAME]/.bundle/debug/default/files has type directory (ID: 0) pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync
+10:07:59 Debug: non-retriable error:  pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
+< {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
+< {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
+< } pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
diff --git a/acceptance/bundle/debug/out.stderr.txt b/acceptance/bundle/debug/out.stderr.txt
new file mode 100644
index 000000000..9cac8bb2b
--- /dev/null
+++ b/acceptance/bundle/debug/out.stderr.txt
@@ -0,0 +1,92 @@
+10:07:59 Info: start pid=12345 version=[DEV_VERSION] args="[CLI], bundle, validate, --debug"
+10:07:59 Debug: Found bundle root at [TMPDIR] (file [TMPDIR]/databricks.yml) pid=12345
+10:07:59 Debug: Apply pid=12345 mutator=load
+10:07:59 Info: Phase: load pid=12345 mutator=load
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=EntryPoint
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=scripts.preinit
+10:07:59 Debug: No script defined for preinit, skipping pid=12345 mutator=load mutator=seq mutator=scripts.preinit
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=ProcessRootIncludes
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=ProcessRootIncludes mutator=seq
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=VerifyCliVersion
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=EnvironmentsToTargets
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=ComputeIdToClusterId
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=InitializeVariables
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=DefineDefaultTarget(default)
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=PythonMutator(load)
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=validate:unique_resource_keys
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=SelectDefaultTarget
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=SelectDefaultTarget mutator=SelectTarget(default)
+10:07:59 Debug: Apply pid=12345 mutator=<func>
+10:07:59 Debug: Apply pid=12345 mutator=initialize
+10:07:59 Info: Phase: initialize pid=12345 mutator=initialize
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=validate:AllResourcesHaveValues
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=RewriteSyncPaths
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SyncDefaultPath
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SyncInferRoot
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PopulateCurrentUser
+10:07:59 Debug: GET /api/2.0/preview/scim/v2/Me
+< HTTP/1.1 200 OK
+< {
+<   "id": "[USERID]",
+<   "userName": "[USERNAME]"
+< } pid=12345 mutator=initialize mutator=seq mutator=PopulateCurrentUser sdk=true
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=LoadGitDetails
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ApplySourceLinkedDeploymentPreset
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=DefineDefaultWorkspaceRoot
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ExpandWorkspaceRoot
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=DefaultWorkspacePaths
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PrependWorkspacePrefix
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=RewriteWorkspacePrefix
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SetVariables
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonMutator(init)
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonMutator(load_resources)
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonMutator(apply_mutators)
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ResolveVariableReferences
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ResolveResourceReferences
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ResolveVariableReferences
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeJobClusters
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeJobParameters
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeJobTasks
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergePipelineClusters
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeApps
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=CaptureSchemaDependency
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=CheckPermissions
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SetRunAs
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=OverrideCompute
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ConfigureDashboardDefaults
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ConfigureVolumeDefaults
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ProcessTargetMode
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ApplyPresets
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=DefaultQueueing
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ExpandPipelineGlobPaths
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ConfigureWSFS
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=TranslatePaths
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonWrapperWarning
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=apps.Validate
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ValidateSharedRootPermissions
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ApplyBundlePermissions
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=FilterCurrentUserFromPermissions
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=metadata.AnnotateJobs
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=metadata.AnnotatePipelines
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize
+10:07:59 Debug: Using Terraform from DATABRICKS_TF_EXEC_PATH at [TERRAFORM] pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize
+10:07:59 Debug: Using Terraform CLI config from DATABRICKS_TF_CLI_CONFIG_FILE at [DATABRICKS_TF_CLI_CONFIG_FILE] pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize
+10:07:59 Debug: Environment variables for Terraform: ...redacted... pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=scripts.postinit
+10:07:59 Debug: No script defined for postinit, skipping pid=12345 mutator=initialize mutator=seq mutator=scripts.postinit
+10:07:59 Debug: Apply pid=12345 mutator=validate
+10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files
+< HTTP/1.1 404 Not Found
+10:07:59 Debug: POST /api/2.0/workspace/mkdirs
+> {
+>   "path": "/Workspace/Users/[USERNAME]/.bundle/debug/default/files"
+> }
+< HTTP/1.1 200 OK
+10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files
+< HTTP/1.1 200 OK
+< {
+<   "object_type": "DIRECTORY",
+<   "path": "/Workspace/Users/[USERNAME]/.bundle/debug/default/files"
+10:07:59 Info: completed execution pid=12345 exit_code=0
diff --git a/acceptance/bundle/debug/output.txt b/acceptance/bundle/debug/output.txt
new file mode 100644
index 000000000..ed72b360e
--- /dev/null
+++ b/acceptance/bundle/debug/output.txt
@@ -0,0 +1,7 @@
+Name: debug
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/debug/default
+
+Validation OK!
diff --git a/acceptance/bundle/debug/script b/acceptance/bundle/debug/script
new file mode 100644
index 000000000..1b4cfc8f0
--- /dev/null
+++ b/acceptance/bundle/debug/script
@@ -0,0 +1,4 @@
+$CLI bundle validate --debug 2> full.stderr.txt
+grep -vw parallel full.stderr.txt > out.stderr.txt
+grep -w parallel full.stderr.txt | sort_lines.py > out.stderr.parallel.txt
+rm full.stderr.txt
diff --git a/acceptance/bundle/debug/test.toml b/acceptance/bundle/debug/test.toml
new file mode 100644
index 000000000..bb0fcb395
--- /dev/null
+++ b/acceptance/bundle/debug/test.toml
@@ -0,0 +1,18 @@
+LocalOnly = true
+
+[[Repls]]
+# The keys are unsorted and also vary per OS
+Old = 'Environment variables for Terraform: ([A-Z_ ,]+) '
+New = 'Environment variables for Terraform: ...redacted... '
+
+[[Repls]]
+Old = 'pid=[0-9]+'
+New = 'pid=12345'
+
+[[Repls]]
+Old = '\d\d:\d\d:\d\d'
+New = '10:07:59'
+
+[[Repls]]
+Old = '\\'
+New = '/'
diff --git a/acceptance/bundle/git-permerror/output.txt b/acceptance/bundle/git-permerror/output.txt
index 03ab93442..730e8255b 100644
--- a/acceptance/bundle/git-permerror/output.txt
+++ b/acceptance/bundle/git-permerror/output.txt
@@ -3,6 +3,7 @@
 >>> chmod 000 .git
 
 >>> [CLI] bundle validate
+Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied
 Error: unable to load repository specific gitconfig: open config: permission denied
 
 Name: git-permerror
@@ -16,6 +17,7 @@ Found 1 error
 Exit code: 1
 
 >>> [CLI] bundle validate -o json
+Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied
 Error: unable to load repository specific gitconfig: open config: permission denied
 
 
@@ -25,6 +27,7 @@ Exit code: 1
 }
 
 >>> withdir subdir/a/b [CLI] bundle validate -o json
+Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied
 Error: unable to load repository specific gitconfig: open config: permission denied
 
 
@@ -39,11 +42,15 @@ Exit code: 1
 >>> chmod 000 .git/HEAD
 
 >>> [CLI] bundle validate -o json
+Warn: failed to load current branch: open HEAD: permission denied
+Warn: failed to load latest commit: open HEAD: permission denied
 {
   "bundle_root_path": "."
 }
 
 >>> withdir subdir/a/b [CLI] bundle validate -o json
+Warn: failed to load current branch: open HEAD: permission denied
+Warn: failed to load latest commit: open HEAD: permission denied
 {
   "bundle_root_path": "."
 }
@@ -54,6 +61,7 @@ Exit code: 1
 >>> chmod 000 .git/config
 
 >>> [CLI] bundle validate -o json
+Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied
 Error: unable to load repository specific gitconfig: open config: permission denied
 
 
@@ -63,6 +71,7 @@ Exit code: 1
 }
 
 >>> withdir subdir/a/b [CLI] bundle validate -o json
+Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied
 Error: unable to load repository specific gitconfig: open config: permission denied
 
 
diff --git a/acceptance/bundle/git-permerror/test.toml b/acceptance/bundle/git-permerror/test.toml
index 3f96e551c..15305cff1 100644
--- a/acceptance/bundle/git-permerror/test.toml
+++ b/acceptance/bundle/git-permerror/test.toml
@@ -1,4 +1,4 @@
-Badness = "Warning logs not shown; inferred flag is set to true incorrect; bundle_root_path is not correct"
+Badness = "inferred flag is set to true incorrect; bundle_root_path is not correct; Warn and Error talk about the same; Warn goes to stderr, Error goes to stdout (for backward compat); Warning about permissions repeated twice"
 
 [GOOS]
 # This test relies on chmod which does not work on Windows
diff --git a/bundle/artifacts/upload.go b/bundle/artifacts/upload.go
index c69939e8c..d4625d85d 100644
--- a/bundle/artifacts/upload.go
+++ b/bundle/artifacts/upload.go
@@ -29,7 +29,7 @@ func (m *cleanUp) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 	// We intentionally ignore the error because it is not critical to the deployment
 	err := client.Delete(ctx, ".", filer.DeleteRecursively)
 	if err != nil {
-		log.Errorf(ctx, "failed to delete %s: %v", uploadPath, err)
+		log.Debugf(ctx, "failed to delete %s: %v", uploadPath, err)
 	}
 
 	err = client.Mkdir(ctx, ".")
diff --git a/bundle/mutator.go b/bundle/mutator.go
index 6c9968aac..16ef79ee7 100644
--- a/bundle/mutator.go
+++ b/bundle/mutator.go
@@ -42,7 +42,7 @@ func Apply(ctx context.Context, b *Bundle, m Mutator) diag.Diagnostics {
 	// such that they are not logged multiple times.
 	// If this is done, we can omit this block.
 	if err := diags.Error(); err != nil {
-		log.Errorf(ctx, "Error: %s", err)
+		log.Debugf(ctx, "Error: %s", err)
 	}
 
 	return diags
diff --git a/bundle/mutator_read_only.go b/bundle/mutator_read_only.go
index ee4e36e0f..700a90d8d 100644
--- a/bundle/mutator_read_only.go
+++ b/bundle/mutator_read_only.go
@@ -22,7 +22,7 @@ func ApplyReadOnly(ctx context.Context, rb ReadOnlyBundle, m ReadOnlyMutator) di
 	log.Debugf(ctx, "ApplyReadOnly")
 	diags := m.Apply(ctx, rb)
 	if err := diags.Error(); err != nil {
-		log.Errorf(ctx, "Error: %s", err)
+		log.Debugf(ctx, "Error: %s", err)
 	}
 
 	return diags
diff --git a/cmd/root/root.go b/cmd/root/root.go
index 3b37d0176..d7adf47f4 100644
--- a/cmd/root/root.go
+++ b/cmd/root/root.go
@@ -114,10 +114,15 @@ func Execute(ctx context.Context, cmd *cobra.Command) error {
 		if err == nil {
 			logger.Info("completed execution",
 				slog.String("exit_code", "0"))
-		} else {
-			logger.Error("failed execution",
+		} else if errors.Is(err, ErrAlreadyPrinted) {
+			logger.Debug("failed execution",
 				slog.String("exit_code", "1"),
-				slog.String("error", err.Error()))
+			)
+		} else {
+			logger.Info("failed execution",
+				slog.String("exit_code", "1"),
+				slog.String("error", err.Error()),
+			)
 		}
 	}
 
diff --git a/libs/flags/log_level_flag.go b/libs/flags/log_level_flag.go
index 836d84b70..82e2abc4c 100644
--- a/libs/flags/log_level_flag.go
+++ b/libs/flags/log_level_flag.go
@@ -25,7 +25,7 @@ type LogLevelFlag struct {
 
 func NewLogLevelFlag() LogLevelFlag {
 	return LogLevelFlag{
-		l: log.LevelDisabled,
+		l: log.LevelWarn,
 	}
 }
 
diff --git a/libs/flags/log_level_flag_test.go b/libs/flags/log_level_flag_test.go
index 11a50bc45..c81f90d18 100644
--- a/libs/flags/log_level_flag_test.go
+++ b/libs/flags/log_level_flag_test.go
@@ -10,8 +10,8 @@ import (
 
 func TestLogLevelFlagDefault(t *testing.T) {
 	f := NewLogLevelFlag()
-	assert.Equal(t, log.LevelDisabled, f.Level())
-	assert.Equal(t, "disabled", f.String())
+	assert.Equal(t, log.LevelWarn, f.Level())
+	assert.Equal(t, "warn", f.String())
 }
 
 func TestLogLevelFlagSetValid(t *testing.T) {
diff --git a/libs/log/handler/friendly.go b/libs/log/handler/friendly.go
index 33b88a9e2..5c60eb13d 100644
--- a/libs/log/handler/friendly.go
+++ b/libs/log/handler/friendly.go
@@ -53,11 +53,11 @@ func NewFriendlyHandler(out io.Writer, opts *Options) slog.Handler {
 
 	// Cache (colorized) level strings.
 	// The colors to use for each level are configured in `colors.go`.
-	h.levelTrace = h.sprintf(ttyColorLevelTrace, "%5s", "TRACE")
-	h.levelDebug = h.sprintf(ttyColorLevelDebug, "%5s", "DEBUG")
-	h.levelInfo = h.sprintf(ttyColorLevelInfo, "%5s", "INFO")
-	h.levelWarn = h.sprintf(ttyColorLevelWarn, "%5s", "WARN")
-	h.levelError = h.sprintf(ttyColorLevelError, "%5s", "ERROR")
+	h.levelTrace = h.sprintf(ttyColorLevelTrace, "%s", "Trace:")
+	h.levelDebug = h.sprintf(ttyColorLevelDebug, "%s", "Debug:")
+	h.levelInfo = h.sprintf(ttyColorLevelInfo, "%s", "Info:")
+	h.levelWarn = h.sprintf(ttyColorLevelWarn, "%s", "Warn:")
+	h.levelError = h.sprintf(ttyColorLevelError, "%s", "Error:")
 	return h
 }
 
@@ -185,33 +185,41 @@ func (s *handleState) appendAttr(a slog.Attr) {
 // Handle implements slog.Handler.
 func (h *friendlyHandler) Handle(ctx context.Context, r slog.Record) error {
 	state := h.handleState()
-	state.append(h.sprintf(ttyColorTime, "%02d:%02d:%02d ", r.Time.Hour(), r.Time.Minute(), r.Time.Second()))
+
+	if h.opts.Level.Level() <= slog.LevelDebug {
+		state.append(h.sprintf(ttyColorTime, "%02d:%02d:%02d ", r.Time.Hour(), r.Time.Minute(), r.Time.Second()))
+	}
+
 	state.appendf("%s ", h.coloredLevel(r))
 	state.append(h.sprint(ttyColorMessage, r.Message))
 
-	// Handle state from WithGroup and WithAttrs.
-	goas := h.goas
-	if r.NumAttrs() == 0 {
-		// If the record has no Attrs, remove groups at the end of the list; they are empty.
-		for len(goas) > 0 && goas[len(goas)-1].group != "" {
-			goas = goas[:len(goas)-1]
-		}
-	}
-	for _, goa := range goas {
-		if goa.group != "" {
-			state.openGroup(goa.group)
-		} else {
-			for _, a := range goa.attrs {
-				state.appendAttr(a)
+	if h.opts.Level.Level() <= slog.LevelDebug {
+
+		// Handle state from WithGroup and WithAttrs.
+		goas := h.goas
+		if r.NumAttrs() == 0 {
+			// If the record has no Attrs, remove groups at the end of the list; they are empty.
+			for len(goas) > 0 && goas[len(goas)-1].group != "" {
+				goas = goas[:len(goas)-1]
+			}
+		}
+		for _, goa := range goas {
+			if goa.group != "" {
+				state.openGroup(goa.group)
+			} else {
+				for _, a := range goa.attrs {
+					state.appendAttr(a)
+				}
 			}
 		}
-	}
 
-	// Add attributes from the record.
-	r.Attrs(func(a slog.Attr) bool {
-		state.appendAttr(a)
-		return true
-	})
+		// Add attributes from the record.
+		r.Attrs(func(a slog.Attr) bool {
+			state.appendAttr(a)
+			return true
+		})
+
+	}
 
 	// Add newline.
 	state.append("\n")

From 65ac9a336a92c3d3f7e0dece648437024a361c90 Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Fri, 7 Feb 2025 17:21:37 +0530
Subject: [PATCH 205/247] Add doc string for the `auth token` command (#2302)

## Changes
The intent of this PR is to clarify that the `databricks auth token`
command is not supported for M2M auth. Fixes:
https://github.com/databricks/cli/issues/1939

## Tests
Manually.
---
 cmd/auth/token.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cmd/auth/token.go b/cmd/auth/token.go
index fbf8b68f6..f3468df40 100644
--- a/cmd/auth/token.go
+++ b/cmd/auth/token.go
@@ -46,6 +46,10 @@ func newTokenCommand(persistentAuth *auth.PersistentAuth) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "token [HOST]",
 		Short: "Get authentication token",
+		Long: `Get authentication token from the local cache in ~/.databricks/token-cache.json.
+Refresh the access token if it is expired. Note: This command only works with
+U2M authentication (using the 'databricks auth login' command). M2M authentication
+using a client ID and secret is not supported.`,
 	}
 
 	var tokenTimeout time.Duration

From ecc05689cafffcbc80ec14e708696ae4beff6629 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 7 Feb 2025 14:13:12 +0100
Subject: [PATCH 206/247] Add a couple of tests for bundle init with custom
 template (#2293)

These test custom template and what happens if helper function returns
an error.
---
 .../helpers-error/databricks_template_schema.json          | 1 +
 acceptance/bundle/templates/helpers-error/output.txt       | 3 +++
 acceptance/bundle/templates/helpers-error/script           | 1 +
 .../templates/helpers-error/template/helpers.txt.tmpl      | 1 +
 acceptance/bundle/templates/helpers-error/test.toml        | 7 +++++++
 .../templates/helpers/databricks_template_schema.json      | 1 +
 acceptance/bundle/templates/helpers/output.txt             | 2 ++
 acceptance/bundle/templates/helpers/script                 | 3 +++
 .../bundle/templates/helpers/template/helpers.txt.tmpl     | 1 +
 acceptance/bundle/templates/helpers/test.toml              | 1 +
 10 files changed, 21 insertions(+)
 create mode 100644 acceptance/bundle/templates/helpers-error/databricks_template_schema.json
 create mode 100644 acceptance/bundle/templates/helpers-error/output.txt
 create mode 100644 acceptance/bundle/templates/helpers-error/script
 create mode 100644 acceptance/bundle/templates/helpers-error/template/helpers.txt.tmpl
 create mode 100644 acceptance/bundle/templates/helpers-error/test.toml
 create mode 100644 acceptance/bundle/templates/helpers/databricks_template_schema.json
 create mode 100644 acceptance/bundle/templates/helpers/output.txt
 create mode 100644 acceptance/bundle/templates/helpers/script
 create mode 100644 acceptance/bundle/templates/helpers/template/helpers.txt.tmpl
 create mode 100644 acceptance/bundle/templates/helpers/test.toml

diff --git a/acceptance/bundle/templates/helpers-error/databricks_template_schema.json b/acceptance/bundle/templates/helpers-error/databricks_template_schema.json
new file mode 100644
index 000000000..0967ef424
--- /dev/null
+++ b/acceptance/bundle/templates/helpers-error/databricks_template_schema.json
@@ -0,0 +1 @@
+{}
diff --git a/acceptance/bundle/templates/helpers-error/output.txt b/acceptance/bundle/templates/helpers-error/output.txt
new file mode 100644
index 000000000..6020e944f
--- /dev/null
+++ b/acceptance/bundle/templates/helpers-error/output.txt
@@ -0,0 +1,3 @@
+Error: failed to compute file content for helpers.txt.tmpl. template: :1:14: executing "" at <user_name>: error calling user_name: 
+
+Exit code: 1
diff --git a/acceptance/bundle/templates/helpers-error/script b/acceptance/bundle/templates/helpers-error/script
new file mode 100644
index 000000000..d9fcbf62c
--- /dev/null
+++ b/acceptance/bundle/templates/helpers-error/script
@@ -0,0 +1 @@
+$CLI bundle init .
diff --git a/acceptance/bundle/templates/helpers-error/template/helpers.txt.tmpl b/acceptance/bundle/templates/helpers-error/template/helpers.txt.tmpl
new file mode 100644
index 000000000..70e60edac
--- /dev/null
+++ b/acceptance/bundle/templates/helpers-error/template/helpers.txt.tmpl
@@ -0,0 +1 @@
+user_name: {{ user_name }}
diff --git a/acceptance/bundle/templates/helpers-error/test.toml b/acceptance/bundle/templates/helpers-error/test.toml
new file mode 100644
index 000000000..77f4ed94b
--- /dev/null
+++ b/acceptance/bundle/templates/helpers-error/test.toml
@@ -0,0 +1,7 @@
+Badness = '''(minor) error message is not great: executing "" at <user_name>: error calling user_name:'''
+LocalOnly = true
+
+[[Server]]
+Pattern = "GET /api/2.0/preview/scim/v2/Me"
+Response.Body = '{}'
+Response.StatusCode = 500
diff --git a/acceptance/bundle/templates/helpers/databricks_template_schema.json b/acceptance/bundle/templates/helpers/databricks_template_schema.json
new file mode 100644
index 000000000..0967ef424
--- /dev/null
+++ b/acceptance/bundle/templates/helpers/databricks_template_schema.json
@@ -0,0 +1 @@
+{}
diff --git a/acceptance/bundle/templates/helpers/output.txt b/acceptance/bundle/templates/helpers/output.txt
new file mode 100644
index 000000000..3126ea5af
--- /dev/null
+++ b/acceptance/bundle/templates/helpers/output.txt
@@ -0,0 +1,2 @@
+✨ Successfully initialized template
+user_name: [USERNAME]
diff --git a/acceptance/bundle/templates/helpers/script b/acceptance/bundle/templates/helpers/script
new file mode 100644
index 000000000..1773e7b03
--- /dev/null
+++ b/acceptance/bundle/templates/helpers/script
@@ -0,0 +1,3 @@
+$CLI bundle init .
+cat helpers.txt
+rm helpers.txt
diff --git a/acceptance/bundle/templates/helpers/template/helpers.txt.tmpl b/acceptance/bundle/templates/helpers/template/helpers.txt.tmpl
new file mode 100644
index 000000000..70e60edac
--- /dev/null
+++ b/acceptance/bundle/templates/helpers/template/helpers.txt.tmpl
@@ -0,0 +1 @@
+user_name: {{ user_name }}
diff --git a/acceptance/bundle/templates/helpers/test.toml b/acceptance/bundle/templates/helpers/test.toml
new file mode 100644
index 000000000..b76e712fb
--- /dev/null
+++ b/acceptance/bundle/templates/helpers/test.toml
@@ -0,0 +1 @@
+LocalOnly = true

From ecb816446e63382cb8871bb376eb26b9f80b29cc Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Fri, 7 Feb 2025 14:54:24 +0000
Subject: [PATCH 207/247] Update app deploy test to confirm app config changes
 on redeploy (#2301)

## Changes
Adds additional step to integration test which changes the app config
and confirms it's updated after redeploy

## Tests
```
    helpers_test.go:156: stderr: Deleting files...
    helpers_test.go:156: stderr: Destroy complete!
--- PASS: TestDeployBundleWithApp (470.25s)
PASS
coverage: [no statements]
ok      github.com/databricks/cli/integration/bundle    470.981s        coverage: [no statements]
```
---
 integration/bundle/apps_test.go                | 18 ++++++++++++++++++
 .../bundles/apps/template/databricks.yml.tmpl  |  6 +++++-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/integration/bundle/apps_test.go b/integration/bundle/apps_test.go
index 01ab52e90..12bd2fcbf 100644
--- a/integration/bundle/apps_test.go
+++ b/integration/bundle/apps_test.go
@@ -98,6 +98,24 @@ func TestDeployBundleWithApp(t *testing.T) {
   - run
 env:
   - name: JOB_ID
+    value: "%d"`, job.JobId))
+
+	// Redeploy bundle with changed config env for app and confirm it's updated in app.yaml
+	deployBundleWithArgs(t, ctx, root, `--var="env_var_name=ANOTHER_JOB_ID"`, "--force-lock", "--auto-approve")
+	reader, err = wt.W.Workspace.Download(ctx, pathToAppYml)
+	require.NoError(t, err)
+
+	data, err = io.ReadAll(reader)
+	require.NoError(t, err)
+
+	content = string(data)
+	require.Contains(t, content, fmt.Sprintf(`command:
+  - flask
+  - --app
+  - app
+  - run
+env:
+  - name: ANOTHER_JOB_ID
     value: "%d"`, job.JobId))
 
 	if testing.Short() {
diff --git a/integration/bundle/bundles/apps/template/databricks.yml.tmpl b/integration/bundle/bundles/apps/template/databricks.yml.tmpl
index 4d862a06f..e0937be71 100644
--- a/integration/bundle/bundles/apps/template/databricks.yml.tmpl
+++ b/integration/bundle/bundles/apps/template/databricks.yml.tmpl
@@ -4,6 +4,10 @@ bundle:
 workspace:
   root_path: "~/.bundle/{{.unique_id}}"
 
+variables:
+  env_var_name:
+    default: "JOB_ID"
+
 resources:
   apps:
     test_app:
@@ -17,7 +21,7 @@ resources:
           - app
           - run
         env:
-          - name: JOB_ID
+          - name: ${var.env_var_name}
             value: ${resources.jobs.foo.id}
 
       resources:

From 6b1a778fe10051d779beb22101825b5a72dcddae Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 7 Feb 2025 17:17:50 +0100
Subject: [PATCH 208/247] Fix flaky acceptance test (#2310)

## Changes
Replace timestamps with fixed string before output is sorted (and before
test runner replacements are applied).

Otherwise the test sometimes fails with error below. Note, timestamps
themselves do not show it, because they were replaced.

```
    --- FAIL: TestAccept/bundle/debug (0.78s)
        acceptance_test.go:404: Diff:
            --- bundle/debug/out.stderr.parallel.txt
            +++ /var/folders/5y/9kkdnjw91p11vsqwk0cvmk200000gp/T/TestAcceptbundledebug1859985035/001/out.stderr.parallel.txt
            @@ -8,8 +8,8 @@
             10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync
             10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:folder_permissions
             10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:validate_sync_patterns
            -10:07:59 Debug: Path /Workspace/Users/[USERNAME]/.bundle/debug/default/files has type directory (ID: 0) pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync
             10:07:59 Debug: non-retriable error:  pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
            +10:07:59 Debug: Path /Workspace/Users/[USERNAME]/.bundle/debug/default/files has type directory (ID: 0) pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync
             < {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
             < {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
             < } pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
```

## Tests
Running `hyperfine --min-runs 10 'go test ../.. -run
^TestAccept$/^bundle$/^debug$ -count=1' --show-output` detects flakiness
on main but not with this PR.
---
 acceptance/bundle/debug/script | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acceptance/bundle/debug/script b/acceptance/bundle/debug/script
index 1b4cfc8f0..005a1a341 100644
--- a/acceptance/bundle/debug/script
+++ b/acceptance/bundle/debug/script
@@ -1,4 +1,4 @@
 $CLI bundle validate --debug 2> full.stderr.txt
 grep -vw parallel full.stderr.txt > out.stderr.txt
-grep -w parallel full.stderr.txt | sort_lines.py > out.stderr.parallel.txt
+grep -w parallel full.stderr.txt | sed 's/[0-9]/0/g' | sort_lines.py > out.stderr.parallel.txt
 rm full.stderr.txt

From f71583fbc09504dbbf0742e53bdcebca53d424a3 Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Fri, 7 Feb 2025 21:56:48 +0530
Subject: [PATCH 209/247] Error when unknown API endpoint is used in testserver
 (#2292)

## Changes
This PR fails the acceptance test when an unknown endpoint (i.e. not
stubbed) is used. We want to ensure that all API endpoints used in an
acceptance test are stubbed and do not otherwise silently fail with a
404.

The logs on failure output include a configuration that developers can
simply copy-paste to `test.toml` to stub the missing API endpoint. It'll
look something like:
```
[[Server]]
Pattern = "<method> <path>"
Response.Body = '''
<response body here>
'''
Response.StatusCode = <response status-code here>
```


## Tests
Manually:

output.txt when an endpoint is not found:
```
>>> [CLI] jobs create --json {"name":"abc"}
Error: No stub found for pattern: POST /api/2.1/jobs/create
```

How this renders in the test logs:
```
    --- FAIL: TestAccept/workspace/jobs/create (0.03s)
        server.go:46:

            ----------------------------------------
            No stub found for pattern: POST /api/2.1/jobs/create

            To stub a response for this request, you can add
            the following to test.toml:
            [[Server]]
            Pattern = "POST /api/2.1/jobs/create"
            Response.Body = '''
            <response body here>
            '''
            Response.StatusCode = <response status-code here>
            ----------------------------------------
```

Manually checked that the debug mode still works.
---
 acceptance/cmd_server_test.go |  5 ++++-
 libs/testserver/server.go     | 34 +++++++++++++++++++++++++++++++++-
 2 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go
index 0166dfe32..c8a52f4cd 100644
--- a/acceptance/cmd_server_test.go
+++ b/acceptance/cmd_server_test.go
@@ -15,7 +15,10 @@ import (
 func StartCmdServer(t *testing.T) *testserver.Server {
 	server := testserver.New(t)
 
-	server.Handle("/", func(w *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	// {$} is a wildcard that only matches the end of the URL. We explicitly use
+	// /{$} to disambiguate it from the generic handler for '/' which is used to
+	// identify unhandled API endpoints in the test server.
+	server.Handle("/{$}", func(w *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		q := r.URL.Query()
 		args := strings.Split(q.Get("args"), " ")
 
diff --git a/libs/testserver/server.go b/libs/testserver/server.go
index ffb83a49c..9ccf34be0 100644
--- a/libs/testserver/server.go
+++ b/libs/testserver/server.go
@@ -12,6 +12,7 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/databricks/cli/internal/testutil"
+	"github.com/databricks/databricks-sdk-go/apierr"
 )
 
 type Server struct {
@@ -41,13 +42,44 @@ func New(t testutil.TestingT) *Server {
 	server := httptest.NewServer(mux)
 	t.Cleanup(server.Close)
 
-	return &Server{
+	s := &Server{
 		Server:         server,
 		Mux:            mux,
 		t:              t,
 		mu:             &sync.Mutex{},
 		fakeWorkspaces: map[string]*FakeWorkspace{},
 	}
+
+	// The server resolves conflicting handlers by using the one with higher
+	// specificity. This handler is the least specific, so it will be used as a
+	// fallback when no other handlers match.
+	s.Handle("/", func(fakeWorkspace *FakeWorkspace, r *http.Request) (any, int) {
+		pattern := r.Method + " " + r.URL.Path
+
+		t.Errorf(`
+
+----------------------------------------
+No stub found for pattern: %s
+
+To stub a response for this request, you can add
+the following to test.toml:
+[[Server]]
+Pattern = %q
+Response.Body = '''
+<response body here>
+'''
+Response.StatusCode = <response status-code here>
+----------------------------------------
+
+
+`, pattern, pattern)
+
+		return apierr.APIError{
+			Message: "No stub found for pattern: " + pattern,
+		}, http.StatusNotFound
+	})
+
+	return s
 }
 
 type HandlerFunc func(fakeWorkspace *FakeWorkspace, req *http.Request) (resp any, statusCode int)

From ff4a5c22698dd31445524672c70f05d05031d4f3 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 7 Feb 2025 17:38:27 +0100
Subject: [PATCH 210/247] acc: Implement config merge  (#2294)

## Changes
Instead of using leaf-most config, all configs from root at
acceptance/test.toml to all intermediate ones to leaf config are merged
into one. Maps are merged, slices are appended, other values are
overridden.

I had to disable caching, because it is tricky when merging is involved
- deep copy is needed. There is performance
impact but currently it is tiny, about 1%.

Also, remove empty root config.

## Tests

Manually checked that inheritance of LocalOnly setting worked for these
tests:

Before - integration tests showed:

```
PASS acceptance.TestAccept/bundle/templates/wrong-url (0.70s)
PASS acceptance.TestAccept/bundle/templates/wrong-path (0.44s)
```

After:

```
SKIP acceptance.TestAccept/bundle/templates/wrong-url (0.00s)
SKIP acceptance.TestAccept/bundle/templates/wrong-path (0.00s)
      acceptance_test.go:216: Disabled via LocalOnly setting in bundle/templates/test.toml, bundle/templates/wrong-path/test.toml (CLOUD_ENV=***)
```
---
 NOTICE                    |  5 ++++
 acceptance/config_test.go | 56 +++++++++++++++++----------------------
 acceptance/test.toml      |  2 --
 go.mod                    |  1 +
 go.sum                    |  4 +--
 5 files changed, 33 insertions(+), 35 deletions(-)
 delete mode 100644 acceptance/test.toml

diff --git a/NOTICE b/NOTICE
index ed22084cf..4331a2a32 100644
--- a/NOTICE
+++ b/NOTICE
@@ -109,3 +109,8 @@ License - https://github.com/hexops/gotextdiff/blob/main/LICENSE
 https://github.com/BurntSushi/toml
 Copyright (c) 2013 TOML authors
 https://github.com/BurntSushi/toml/blob/master/COPYING
+
+dario.cat/mergo
+Copyright (c) 2013 Dario Castañé. All rights reserved.
+Copyright (c) 2012 The Go Authors. All rights reserved.
+https://github.com/darccio/mergo/blob/master/LICENSE
diff --git a/acceptance/config_test.go b/acceptance/config_test.go
index e24a683e7..920e713a1 100644
--- a/acceptance/config_test.go
+++ b/acceptance/config_test.go
@@ -3,9 +3,11 @@ package acceptance_test
 import (
 	"os"
 	"path/filepath"
-	"sync"
+	"slices"
+	"strings"
 	"testing"
 
+	"dario.cat/mergo"
 	"github.com/BurntSushi/toml"
 	"github.com/databricks/cli/libs/testdiff"
 	"github.com/stretchr/testify/require"
@@ -13,11 +15,6 @@ import (
 
 const configFilename = "test.toml"
 
-var (
-	configCache map[string]TestConfig
-	configMutex sync.Mutex
-)
-
 type TestConfig struct {
 	// Place to describe what's wrong with this test. Does not affect how the test is run.
 	Badness string
@@ -65,58 +62,55 @@ type ServerStub struct {
 	}
 }
 
-// FindConfig finds the closest config file.
-func FindConfig(t *testing.T, dir string) (string, bool) {
-	shared := false
+// FindConfigs finds all the config relevant for this test,
+// ordered from the most outermost (at acceptance/) to current test directory (identified by dir).
+// Argument dir must be a relative path from the root of acceptance tests (<project_root>/acceptance/).
+func FindConfigs(t *testing.T, dir string) []string {
+	configs := []string{}
 	for {
 		path := filepath.Join(dir, configFilename)
 		_, err := os.Stat(path)
 
 		if err == nil {
-			return path, shared
+			configs = append(configs, path)
 		}
 
-		shared = true
-
 		if dir == "" || dir == "." {
 			break
 		}
 
-		if os.IsNotExist(err) {
-			dir = filepath.Dir(dir)
+		dir = filepath.Dir(dir)
+
+		if err == nil || os.IsNotExist(err) {
 			continue
 		}
 
 		t.Fatalf("Error while reading %s: %s", path, err)
 	}
 
-	t.Fatal("Config not found: " + configFilename)
-	return "", shared
+	slices.Reverse(configs)
+	return configs
 }
 
 // LoadConfig loads the config file. Non-leaf configs are cached.
 func LoadConfig(t *testing.T, dir string) (TestConfig, string) {
-	path, leafConfig := FindConfig(t, dir)
+	configs := FindConfigs(t, dir)
 
-	if leafConfig {
-		return DoLoadConfig(t, path), path
+	if len(configs) == 0 {
+		return TestConfig{}, "(no config)"
 	}
 
-	configMutex.Lock()
-	defer configMutex.Unlock()
+	result := DoLoadConfig(t, configs[0])
 
-	if configCache == nil {
-		configCache = make(map[string]TestConfig)
+	for _, cfgName := range configs[1:] {
+		cfg := DoLoadConfig(t, cfgName)
+		err := mergo.Merge(&result, cfg, mergo.WithOverride, mergo.WithAppendSlice)
+		if err != nil {
+			t.Fatalf("Error during config merge: %s: %s", cfgName, err)
+		}
 	}
 
-	result, ok := configCache[path]
-	if ok {
-		return result, path
-	}
-
-	result = DoLoadConfig(t, path)
-	configCache[path] = result
-	return result, path
+	return result, strings.Join(configs, ", ")
 }
 
 func DoLoadConfig(t *testing.T, path string) TestConfig {
diff --git a/acceptance/test.toml b/acceptance/test.toml
deleted file mode 100644
index eee94d0ea..000000000
--- a/acceptance/test.toml
+++ /dev/null
@@ -1,2 +0,0 @@
-# If test directory nor any of its parents do not have test.toml then this file serves as fallback configuration.
-# The configurations are not merged across parents; the closest one is used fully.
diff --git a/go.mod b/go.mod
index b3f11e918..662fcd40b 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,7 @@ go 1.23
 toolchain go1.23.4
 
 require (
+	dario.cat/mergo v1.0.1 // BSD 3-Clause
 	github.com/BurntSushi/toml v1.4.0 // MIT
 	github.com/Masterminds/semver/v3 v3.3.1 // MIT
 	github.com/briandowns/spinner v1.23.1 // Apache 2.0
diff --git a/go.sum b/go.sum
index 4e295a82d..bffc3b53d 100644
--- a/go.sum
+++ b/go.sum
@@ -5,8 +5,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKF
 cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q=
 cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
 cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
-dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
-dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
+dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s=
+dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
 github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=

From 5aa89230e96635efaec5aa32a6a06a8cab9dec2f Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Fri, 7 Feb 2025 17:22:51 +0000
Subject: [PATCH 211/247] Use CreatePipeline instead of PipelineSpec for
 resources.Pipeline struct (#2287)

## Changes
`CreatePipeline` is a more complete structure (superset of PipelineSpec
one) which enables support of additional fields such as `run_as` and
`allow_duplicate_names` in DABs configuration. Note: these fields are
subject to support in TF in order to correctly work.

## Tests
Existing tests pass + no fields are removed from JSON schema
---
 bundle/config/mutator/apply_presets.go        |  2 +-
 .../mutator/capture_schema_dependency.go      |  4 +--
 .../mutator/capture_schema_dependency_test.go | 30 +++++++++----------
 .../expand_pipeline_glob_paths_test.go        |  2 +-
 bundle/config/mutator/initialize_urls_test.go |  4 +--
 .../mutator/merge_pipeline_clusters_test.go   |  4 +--
 .../mutator/process_target_mode_test.go       | 10 +++----
 .../resolve_variable_references_test.go       |  6 ++--
 bundle/config/mutator/translate_paths_test.go | 16 +++++-----
 bundle/config/resources/pipeline.go           |  4 +--
 .../validate/single_node_cluster_test.go      |  4 +--
 bundle/deploy/metadata/annotate_pipelines.go  |  4 +--
 .../metadata/annotate_pipelines_test.go       |  8 ++---
 bundle/deploy/terraform/convert_test.go       |  8 ++---
 .../terraform/tfdyn/convert_pipeline.go       |  5 ++++
 .../terraform/tfdyn/convert_pipeline_test.go  | 11 ++++++-
 bundle/internal/schema/annotations.yml        | 10 +++++++
 .../internal/schema/annotations_openapi.yml   | 17 +++++++++++
 .../schema/annotations_openapi_overrides.yml  |  6 ++++
 bundle/internal/schema/main.go                | 15 ++++++++++
 bundle/permissions/workspace_root_test.go     |  8 ++---
 bundle/render/render_text_output_test.go      |  8 ++---
 bundle/resources/completion_test.go           |  4 +--
 bundle/resources/lookup_test.go               |  4 +--
 bundle/run/pipeline.go                        |  4 +--
 bundle/schema/jsonschema.json                 | 26 ++++++++++++++++
 cmd/bundle/generate/pipeline.go               |  2 +-
 libs/dyn/drop_keys.go                         | 27 +++++++++++++++++
 libs/dyn/drop_keys_test.go                    | 24 +++++++++++++++
 29 files changed, 208 insertions(+), 69 deletions(-)
 create mode 100644 libs/dyn/drop_keys.go
 create mode 100644 libs/dyn/drop_keys_test.go

diff --git a/bundle/config/mutator/apply_presets.go b/bundle/config/mutator/apply_presets.go
index b402053e7..c8e7bf9e8 100644
--- a/bundle/config/mutator/apply_presets.go
+++ b/bundle/config/mutator/apply_presets.go
@@ -84,7 +84,7 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 
 	// Pipelines presets: Prefix, PipelinesDevelopment
 	for key, p := range r.Pipelines {
-		if p.PipelineSpec == nil {
+		if p.CreatePipeline == nil {
 			diags = diags.Extend(diag.Errorf("pipeline %s is not defined", key))
 			continue
 		}
diff --git a/bundle/config/mutator/capture_schema_dependency.go b/bundle/config/mutator/capture_schema_dependency.go
index 5025c9a0d..2e17a8175 100644
--- a/bundle/config/mutator/capture_schema_dependency.go
+++ b/bundle/config/mutator/capture_schema_dependency.go
@@ -56,7 +56,7 @@ func resolveVolume(v *resources.Volume, b *bundle.Bundle) {
 }
 
 func resolvePipelineSchema(p *resources.Pipeline, b *bundle.Bundle) {
-	if p == nil || p.PipelineSpec == nil {
+	if p == nil || p.CreatePipeline == nil {
 		return
 	}
 	if p.Schema == "" {
@@ -71,7 +71,7 @@ func resolvePipelineSchema(p *resources.Pipeline, b *bundle.Bundle) {
 }
 
 func resolvePipelineTarget(p *resources.Pipeline, b *bundle.Bundle) {
-	if p == nil || p.PipelineSpec == nil {
+	if p == nil || p.CreatePipeline == nil {
 		return
 	}
 	if p.Target == "" {
diff --git a/bundle/config/mutator/capture_schema_dependency_test.go b/bundle/config/mutator/capture_schema_dependency_test.go
index 0a94e7748..16fa636ee 100644
--- a/bundle/config/mutator/capture_schema_dependency_test.go
+++ b/bundle/config/mutator/capture_schema_dependency_test.go
@@ -118,43 +118,43 @@ func TestCaptureSchemaDependencyForPipelinesWithTarget(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline1": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalog1",
 							Schema:  "foobar",
 						},
 					},
 					"pipeline2": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalog2",
 							Schema:  "foobar",
 						},
 					},
 					"pipeline3": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalog1",
 							Schema:  "barfoo",
 						},
 					},
 					"pipeline4": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalogX",
 							Schema:  "foobar",
 						},
 					},
 					"pipeline5": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalog1",
 							Schema:  "schemaX",
 						},
 					},
 					"pipeline6": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "",
 							Schema:  "foobar",
 						},
 					},
 					"pipeline7": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "",
 							Schema:  "",
 							Name:    "whatever",
@@ -179,7 +179,7 @@ func TestCaptureSchemaDependencyForPipelinesWithTarget(t *testing.T) {
 	assert.Equal(t, "", b.Config.Resources.Pipelines["pipeline7"].Schema)
 
 	assert.Nil(t, b.Config.Resources.Pipelines["nilPipeline"])
-	assert.Nil(t, b.Config.Resources.Pipelines["emptyPipeline"].PipelineSpec)
+	assert.Nil(t, b.Config.Resources.Pipelines["emptyPipeline"].CreatePipeline)
 
 	for _, k := range []string{"pipeline1", "pipeline2", "pipeline3", "pipeline4", "pipeline5", "pipeline6", "pipeline7"} {
 		assert.Empty(t, b.Config.Resources.Pipelines[k].Target)
@@ -214,43 +214,43 @@ func TestCaptureSchemaDependencyForPipelinesWithSchema(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline1": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalog1",
 							Target:  "foobar",
 						},
 					},
 					"pipeline2": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalog2",
 							Target:  "foobar",
 						},
 					},
 					"pipeline3": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalog1",
 							Target:  "barfoo",
 						},
 					},
 					"pipeline4": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalogX",
 							Target:  "foobar",
 						},
 					},
 					"pipeline5": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalog1",
 							Target:  "schemaX",
 						},
 					},
 					"pipeline6": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "",
 							Target:  "foobar",
 						},
 					},
 					"pipeline7": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "",
 							Target:  "",
 							Name:    "whatever",
diff --git a/bundle/config/mutator/expand_pipeline_glob_paths_test.go b/bundle/config/mutator/expand_pipeline_glob_paths_test.go
index 7cf3c9f3e..c5b1ad39d 100644
--- a/bundle/config/mutator/expand_pipeline_glob_paths_test.go
+++ b/bundle/config/mutator/expand_pipeline_glob_paths_test.go
@@ -47,7 +47,7 @@ func TestExpandGlobPathsInPipelines(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Libraries: []pipelines.PipelineLibrary{
 								{
 									Notebook: &pipelines.NotebookLibrary{
diff --git a/bundle/config/mutator/initialize_urls_test.go b/bundle/config/mutator/initialize_urls_test.go
index f07a7deb3..8c751079b 100644
--- a/bundle/config/mutator/initialize_urls_test.go
+++ b/bundle/config/mutator/initialize_urls_test.go
@@ -31,8 +31,8 @@ func TestInitializeURLs(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline1": {
-						ID:           "3",
-						PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1"},
+						ID:             "3",
+						CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline1"},
 					},
 				},
 				Experiments: map[string]*resources.MlflowExperiment{
diff --git a/bundle/config/mutator/merge_pipeline_clusters_test.go b/bundle/config/mutator/merge_pipeline_clusters_test.go
index f117d9399..97ec44eea 100644
--- a/bundle/config/mutator/merge_pipeline_clusters_test.go
+++ b/bundle/config/mutator/merge_pipeline_clusters_test.go
@@ -19,7 +19,7 @@ func TestMergePipelineClusters(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"foo": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Clusters: []pipelines.PipelineCluster{
 								{
 									NodeTypeId: "i3.xlarge",
@@ -68,7 +68,7 @@ func TestMergePipelineClustersCaseInsensitive(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"foo": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Clusters: []pipelines.PipelineCluster{
 								{
 									Label:      "default",
diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go
index 723b01ee3..6a0fd8e03 100644
--- a/bundle/config/mutator/process_target_mode_test.go
+++ b/bundle/config/mutator/process_target_mode_test.go
@@ -88,7 +88,7 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
 					},
 				},
 				Pipelines: map[string]*resources.Pipeline{
-					"pipeline1": {PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1", Continuous: true}},
+					"pipeline1": {CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline1", Continuous: true}},
 				},
 				Experiments: map[string]*resources.MlflowExperiment{
 					"experiment1": {Experiment: &ml.Experiment{Name: "/Users/lennart.kats@databricks.com/experiment1"}},
@@ -181,7 +181,7 @@ func TestProcessTargetModeDevelopment(t *testing.T) {
 	// Pipeline 1
 	assert.Equal(t, "[dev lennart] pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name)
 	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].Continuous)
-	assert.True(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
+	assert.True(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development)
 
 	// Experiment 1
 	assert.Equal(t, "/Users/lennart.kats@databricks.com/[dev lennart] experiment1", b.Config.Resources.Experiments["experiment1"].Name)
@@ -316,7 +316,7 @@ func TestProcessTargetModeDefault(t *testing.T) {
 	require.NoError(t, diags.Error())
 	assert.Equal(t, "job1", b.Config.Resources.Jobs["job1"].Name)
 	assert.Equal(t, "pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name)
-	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
+	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development)
 	assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name)
 	assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name)
 	assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName)
@@ -362,7 +362,7 @@ func TestProcessTargetModeProduction(t *testing.T) {
 
 	assert.Equal(t, "job1", b.Config.Resources.Jobs["job1"].Name)
 	assert.Equal(t, "pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name)
-	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
+	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development)
 	assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name)
 	assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name)
 	assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName)
@@ -568,5 +568,5 @@ func TestPipelinesDevelopmentDisabled(t *testing.T) {
 	diags := bundle.Apply(context.Background(), b, m)
 	require.NoError(t, diags.Error())
 
-	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
+	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development)
 }
diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go
index 44f6c8dbb..30969dc49 100644
--- a/bundle/config/mutator/resolve_variable_references_test.go
+++ b/bundle/config/mutator/resolve_variable_references_test.go
@@ -20,7 +20,7 @@ func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) {
 			true,
 			func(t *testing.T, b *bundle.Bundle) {
 				// Variables that use workspace file path should have SyncRootValue during resolution phase
-				require.Equal(t, "sync/root/path", b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Configuration["source"])
+				require.Equal(t, "sync/root/path", b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Configuration["source"])
 
 				// The file path itself should remain the same
 				require.Equal(t, "file/path", b.Config.Workspace.FilePath)
@@ -29,7 +29,7 @@ func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) {
 		{
 			false,
 			func(t *testing.T, b *bundle.Bundle) {
-				require.Equal(t, "file/path", b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Configuration["source"])
+				require.Equal(t, "file/path", b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Configuration["source"])
 				require.Equal(t, "file/path", b.Config.Workspace.FilePath)
 			},
 		},
@@ -48,7 +48,7 @@ func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) {
 				Resources: config.Resources{
 					Pipelines: map[string]*resources.Pipeline{
 						"pipeline1": {
-							PipelineSpec: &pipelines.PipelineSpec{
+							CreatePipeline: &pipelines.CreatePipeline{
 								Configuration: map[string]string{
 									"source": "${workspace.file_path}",
 								},
diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go
index aa6488ab0..6cfe5718a 100644
--- a/bundle/config/mutator/translate_paths_test.go
+++ b/bundle/config/mutator/translate_paths_test.go
@@ -179,7 +179,7 @@ func TestTranslatePaths(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Libraries: []pipelines.PipelineLibrary{
 								{
 									Notebook: &pipelines.NotebookLibrary{
@@ -333,7 +333,7 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Libraries: []pipelines.PipelineLibrary{
 								{
 									File: &pipelines.FileLibrary{
@@ -488,7 +488,7 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Libraries: []pipelines.PipelineLibrary{
 								{
 									Notebook: &pipelines.NotebookLibrary{
@@ -532,7 +532,7 @@ func TestPipelineNotebookDoesNotExistErrorWithoutExtension(t *testing.T) {
 					Resources: config.Resources{
 						Pipelines: map[string]*resources.Pipeline{
 							"pipeline": {
-								PipelineSpec: &pipelines.PipelineSpec{
+								CreatePipeline: &pipelines.CreatePipeline{
 									Libraries: []pipelines.PipelineLibrary{
 										{
 											Notebook: &pipelines.NotebookLibrary{
@@ -572,7 +572,7 @@ func TestPipelineFileDoesNotExistError(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Libraries: []pipelines.PipelineLibrary{
 								{
 									File: &pipelines.FileLibrary{
@@ -677,7 +677,7 @@ func TestPipelineNotebookLibraryWithFileSourceError(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Libraries: []pipelines.PipelineLibrary{
 								{
 									Notebook: &pipelines.NotebookLibrary{
@@ -712,7 +712,7 @@ func TestPipelineFileLibraryWithNotebookSourceError(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Libraries: []pipelines.PipelineLibrary{
 								{
 									File: &pipelines.FileLibrary{
@@ -916,7 +916,7 @@ func TestTranslatePathsWithSourceLinkedDeployment(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Libraries: []pipelines.PipelineLibrary{
 								{
 									Notebook: &pipelines.NotebookLibrary{
diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go
index 5127d07ba..57d9c4f19 100644
--- a/bundle/config/resources/pipeline.go
+++ b/bundle/config/resources/pipeline.go
@@ -16,7 +16,7 @@ type Pipeline struct {
 	ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
 	URL            string         `json:"url,omitempty" bundle:"internal"`
 
-	*pipelines.PipelineSpec
+	*pipelines.CreatePipeline
 }
 
 func (s *Pipeline) UnmarshalJSON(b []byte) error {
@@ -59,5 +59,5 @@ func (s *Pipeline) GetURL() string {
 }
 
 func (s *Pipeline) IsNil() bool {
-	return s.PipelineSpec == nil
+	return s.CreatePipeline == nil
 }
diff --git a/bundle/config/validate/single_node_cluster_test.go b/bundle/config/validate/single_node_cluster_test.go
index c3ead8ef6..be93420c6 100644
--- a/bundle/config/validate/single_node_cluster_test.go
+++ b/bundle/config/validate/single_node_cluster_test.go
@@ -238,7 +238,7 @@ func TestValidateSingleNodeClusterFailForPipelineClusters(t *testing.T) {
 					Resources: config.Resources{
 						Pipelines: map[string]*resources.Pipeline{
 							"foo": {
-								PipelineSpec: &pipelines.PipelineSpec{
+								CreatePipeline: &pipelines.CreatePipeline{
 									Clusters: []pipelines.PipelineCluster{
 										{
 											SparkConf:  tc.sparkConf,
@@ -493,7 +493,7 @@ func TestValidateSingleNodeClusterPassPipelineClusters(t *testing.T) {
 					Resources: config.Resources{
 						Pipelines: map[string]*resources.Pipeline{
 							"foo": {
-								PipelineSpec: &pipelines.PipelineSpec{
+								CreatePipeline: &pipelines.CreatePipeline{
 									Clusters: []pipelines.PipelineCluster{
 										{
 											SparkConf:  tc.sparkConf,
diff --git a/bundle/deploy/metadata/annotate_pipelines.go b/bundle/deploy/metadata/annotate_pipelines.go
index 990f48907..407aaea6e 100644
--- a/bundle/deploy/metadata/annotate_pipelines.go
+++ b/bundle/deploy/metadata/annotate_pipelines.go
@@ -20,11 +20,11 @@ func (m *annotatePipelines) Name() string {
 
 func (m *annotatePipelines) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
 	for _, pipeline := range b.Config.Resources.Pipelines {
-		if pipeline.PipelineSpec == nil {
+		if pipeline.CreatePipeline == nil {
 			continue
 		}
 
-		pipeline.PipelineSpec.Deployment = &pipelines.PipelineDeployment{
+		pipeline.CreatePipeline.Deployment = &pipelines.PipelineDeployment{
 			Kind:             pipelines.DeploymentKindBundle,
 			MetadataFilePath: metadataFilePath(b),
 		}
diff --git a/bundle/deploy/metadata/annotate_pipelines_test.go b/bundle/deploy/metadata/annotate_pipelines_test.go
index 448a022d0..606292724 100644
--- a/bundle/deploy/metadata/annotate_pipelines_test.go
+++ b/bundle/deploy/metadata/annotate_pipelines_test.go
@@ -21,12 +21,12 @@ func TestAnnotatePipelinesMutator(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"my-pipeline-1": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Name: "My Pipeline One",
 						},
 					},
 					"my-pipeline-2": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Name: "My Pipeline Two",
 						},
 					},
@@ -43,14 +43,14 @@ func TestAnnotatePipelinesMutator(t *testing.T) {
 			Kind:             pipelines.DeploymentKindBundle,
 			MetadataFilePath: "/a/b/c/metadata.json",
 		},
-		b.Config.Resources.Pipelines["my-pipeline-1"].PipelineSpec.Deployment)
+		b.Config.Resources.Pipelines["my-pipeline-1"].CreatePipeline.Deployment)
 
 	assert.Equal(t,
 		&pipelines.PipelineDeployment{
 			Kind:             pipelines.DeploymentKindBundle,
 			MetadataFilePath: "/a/b/c/metadata.json",
 		},
-		b.Config.Resources.Pipelines["my-pipeline-2"].PipelineSpec.Deployment)
+		b.Config.Resources.Pipelines["my-pipeline-2"].CreatePipeline.Deployment)
 }
 
 func TestAnnotatePipelinesMutatorPipelineWithoutASpec(t *testing.T) {
diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go
index afc1fb22a..53d861b32 100644
--- a/bundle/deploy/terraform/convert_test.go
+++ b/bundle/deploy/terraform/convert_test.go
@@ -203,7 +203,7 @@ func TestBundleToTerraformForEachTaskLibraries(t *testing.T) {
 
 func TestBundleToTerraformPipeline(t *testing.T) {
 	src := resources.Pipeline{
-		PipelineSpec: &pipelines.PipelineSpec{
+		CreatePipeline: &pipelines.CreatePipeline{
 			Name: "my pipeline",
 			Libraries: []pipelines.PipelineLibrary{
 				{
@@ -759,7 +759,7 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
 			},
 			Pipelines: map[string]*resources.Pipeline{
 				"test_pipeline": {
-					PipelineSpec: &pipelines.PipelineSpec{
+					CreatePipeline: &pipelines.CreatePipeline{
 						Name: "test_pipeline",
 					},
 				},
@@ -898,12 +898,12 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
 			},
 			Pipelines: map[string]*resources.Pipeline{
 				"test_pipeline": {
-					PipelineSpec: &pipelines.PipelineSpec{
+					CreatePipeline: &pipelines.CreatePipeline{
 						Name: "test_pipeline",
 					},
 				},
 				"test_pipeline_new": {
-					PipelineSpec: &pipelines.PipelineSpec{
+					CreatePipeline: &pipelines.CreatePipeline{
 						Name: "test_pipeline_new",
 					},
 				},
diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline.go b/bundle/deploy/terraform/tfdyn/convert_pipeline.go
index ea0c94d66..53a986864 100644
--- a/bundle/deploy/terraform/tfdyn/convert_pipeline.go
+++ b/bundle/deploy/terraform/tfdyn/convert_pipeline.go
@@ -21,6 +21,11 @@ func convertPipelineResource(ctx context.Context, vin dyn.Value) (dyn.Value, err
 		return dyn.InvalidValue, err
 	}
 
+	vout, err = dyn.DropKeys(vout, []string{"allow_duplicate_names", "dry_run"})
+	if err != nil {
+		return dyn.InvalidValue, err
+	}
+
 	// Normalize the output value to the target schema.
 	vout, diags := convert.Normalize(schema.ResourcePipeline{}, vout)
 	for _, diag := range diags {
diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go b/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go
index 0239bad18..d8de55bf0 100644
--- a/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go
+++ b/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go
@@ -15,8 +15,17 @@ import (
 
 func TestConvertPipeline(t *testing.T) {
 	src := resources.Pipeline{
-		PipelineSpec: &pipelines.PipelineSpec{
+		CreatePipeline: &pipelines.CreatePipeline{
 			Name: "my pipeline",
+			// This fields is not part of TF schema yet, but once we upgrade to TF version that supports it, this test will fail because run_as
+			// will be exposed which is expected and test will need to be updated.
+			RunAs: &pipelines.RunAs{
+				UserName: "foo@bar.com",
+			},
+			// We expect AllowDuplicateNames and DryRun to be ignored and not passed to the TF output.
+			// This is not supported by TF now, so we don't want to expose it.
+			AllowDuplicateNames: true,
+			DryRun:              true,
 			Libraries: []pipelines.PipelineLibrary{
 				{
 					Notebook: &pipelines.NotebookLibrary{
diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml
index c10f43b04..2d1a6a3d8 100644
--- a/bundle/internal/schema/annotations.yml
+++ b/bundle/internal/schema/annotations.yml
@@ -414,6 +414,16 @@ github.com/databricks/cli/bundle/config/resources.Permission:
   "user_name":
     "description": |-
       The name of the user that has the permission set in level.
+github.com/databricks/cli/bundle/config/resources.Pipeline:
+  "allow_duplicate_names":
+    "description": |-
+      PLACEHOLDER
+  "dry_run":
+    "description": |-
+      PLACEHOLDER
+  "run_as":
+    "description": |-
+      PLACEHOLDER
 github.com/databricks/cli/bundle/config/variable.Lookup:
   "alert":
     "description": |-
diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml
index d5a9bf69e..d9a0be50e 100644
--- a/bundle/internal/schema/annotations_openapi.yml
+++ b/bundle/internal/schema/annotations_openapi.yml
@@ -371,6 +371,9 @@ github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint:
     "description": |-
       Tags to be attached to the serving endpoint and automatically propagated to billing logs.
 github.com/databricks/cli/bundle/config/resources.Pipeline:
+  "allow_duplicate_names":
+    "description": |-
+      If false, deployment will fail if name conflicts with that of another pipeline.
   "budget_policy_id":
     "description": |-
       Budget policy of this pipeline.
@@ -395,6 +398,7 @@ github.com/databricks/cli/bundle/config/resources.Pipeline:
   "development":
     "description": |-
       Whether the pipeline is in Development mode. Defaults to false.
+  "dry_run": {}
   "edition":
     "description": |-
       Pipeline product edition.
@@ -425,6 +429,7 @@ github.com/databricks/cli/bundle/config/resources.Pipeline:
   "restart_window":
     "description": |-
       Restart window of this pipeline.
+  "run_as": {}
   "schema":
     "description": |-
       The default schema (database) where tables are read from or published to. The presence of this field implies that the pipeline is in direct publishing mode.
@@ -2624,6 +2629,18 @@ github.com/databricks/databricks-sdk-go/service/pipelines.RestartWindow:
     "description": |-
       Time zone id of restart window. See https://docs.databricks.com/sql/language-manual/sql-ref-syntax-aux-conf-mgmt-set-timezone.html for details.
       If not specified, UTC will be used.
+github.com/databricks/databricks-sdk-go/service/pipelines.RunAs:
+  "_":
+    "description": |-
+      Write-only setting, available only in Create/Update calls. Specifies the user or service principal that the pipeline runs as. If not specified, the pipeline runs as the user who created the pipeline.
+
+      Only `user_name` or `service_principal_name` can be specified. If both are specified, an error is thrown.
+  "service_principal_name":
+    "description": |-
+      Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.
+  "user_name":
+    "description": |-
+      The email of an active workspace user. Users can only set this field to their own email.
 github.com/databricks/databricks-sdk-go/service/pipelines.SchemaSpec:
   "destination_catalog":
     "description": |-
diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml
index 585886313..be83af2d1 100644
--- a/bundle/internal/schema/annotations_openapi_overrides.yml
+++ b/bundle/internal/schema/annotations_openapi_overrides.yml
@@ -239,9 +239,15 @@ github.com/databricks/cli/bundle/config/resources.Pipeline:
               - notebook:
                   path: ./pipeline.py
       ```
+  "dry_run":
+    "description": |-
+      PLACEHOLDER
   "permissions":
     "description": |-
       PLACEHOLDER
+  "run_as":
+    "description": |-
+      PLACEHOLDER
 github.com/databricks/cli/bundle/config/resources.QualityMonitor:
   "_":
     "markdown_description": |-
diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go
index 38e099ece..2e0120e62 100644
--- a/bundle/internal/schema/main.go
+++ b/bundle/internal/schema/main.go
@@ -109,6 +109,20 @@ func removeJobsFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
 	return s
 }
 
+func removePipelineFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
+	switch typ {
+	case reflect.TypeOf(resources.Pipeline{}):
+		// Even though DABs supports this field, TF provider does not. Thus, we
+		// should not expose it to the user.
+		delete(s.Properties, "dry_run")
+		delete(s.Properties, "allow_duplicate_names")
+	default:
+		// Do nothing
+	}
+
+	return s
+}
+
 // While volume_type is required in the volume create API, DABs automatically sets
 // it's value to "MANAGED" if it's not provided. Thus, we make it optional
 // in the bundle schema.
@@ -168,6 +182,7 @@ func generateSchema(workdir, outputFile string) {
 	// Generate the JSON schema from the bundle Go struct.
 	s, err := jsonschema.FromType(reflect.TypeOf(config.Root{}), []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{
 		removeJobsFields,
+		removePipelineFields,
 		makeVolumeTypeOptional,
 		a.addAnnotations,
 		addInterpolationPatterns,
diff --git a/bundle/permissions/workspace_root_test.go b/bundle/permissions/workspace_root_test.go
index c48704a63..3e5f9c61b 100644
--- a/bundle/permissions/workspace_root_test.go
+++ b/bundle/permissions/workspace_root_test.go
@@ -38,8 +38,8 @@ func TestApplyWorkspaceRootPermissions(t *testing.T) {
 					"job_2": {JobSettings: &jobs.JobSettings{Name: "job_2"}},
 				},
 				Pipelines: map[string]*resources.Pipeline{
-					"pipeline_1": {PipelineSpec: &pipelines.PipelineSpec{}},
-					"pipeline_2": {PipelineSpec: &pipelines.PipelineSpec{}},
+					"pipeline_1": {CreatePipeline: &pipelines.CreatePipeline{}},
+					"pipeline_2": {CreatePipeline: &pipelines.CreatePipeline{}},
 				},
 				Models: map[string]*resources.MlflowModel{
 					"model_1": {Model: &ml.Model{}},
@@ -98,8 +98,8 @@ func TestApplyWorkspaceRootPermissionsForAllPaths(t *testing.T) {
 					"job_2": {JobSettings: &jobs.JobSettings{Name: "job_2"}},
 				},
 				Pipelines: map[string]*resources.Pipeline{
-					"pipeline_1": {PipelineSpec: &pipelines.PipelineSpec{}},
-					"pipeline_2": {PipelineSpec: &pipelines.PipelineSpec{}},
+					"pipeline_1": {CreatePipeline: &pipelines.CreatePipeline{}},
+					"pipeline_2": {CreatePipeline: &pipelines.CreatePipeline{}},
 				},
 				Models: map[string]*resources.MlflowModel{
 					"model_1": {Model: &ml.Model{}},
diff --git a/bundle/render/render_text_output_test.go b/bundle/render/render_text_output_test.go
index 506756f70..d092e77c8 100644
--- a/bundle/render/render_text_output_test.go
+++ b/bundle/render/render_text_output_test.go
@@ -530,12 +530,12 @@ func TestRenderSummary(t *testing.T) {
 					"pipeline2": {
 						ID: "4",
 						// no URL
-						PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline2-name"},
+						CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline2-name"},
 					},
 					"pipeline1": {
-						ID:           "3",
-						URL:          "https://url3",
-						PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1-name"},
+						ID:             "3",
+						URL:            "https://url3",
+						CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline1-name"},
 					},
 				},
 				Schemas: map[string]*resources.Schema{
diff --git a/bundle/resources/completion_test.go b/bundle/resources/completion_test.go
index 80412b6f1..56559f18c 100644
--- a/bundle/resources/completion_test.go
+++ b/bundle/resources/completion_test.go
@@ -25,7 +25,7 @@ func TestCompletions_SkipDuplicates(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"foo": {
-						PipelineSpec: &pipelines.PipelineSpec{},
+						CreatePipeline: &pipelines.CreatePipeline{},
 					},
 				},
 			},
@@ -50,7 +50,7 @@ func TestCompletions_Filter(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"bar": {
-						PipelineSpec: &pipelines.PipelineSpec{},
+						CreatePipeline: &pipelines.CreatePipeline{},
 					},
 				},
 			},
diff --git a/bundle/resources/lookup_test.go b/bundle/resources/lookup_test.go
index 0ea5af7a2..d95da977a 100644
--- a/bundle/resources/lookup_test.go
+++ b/bundle/resources/lookup_test.go
@@ -56,7 +56,7 @@ func TestLookup_MultipleFound(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"foo": {
-						PipelineSpec: &pipelines.PipelineSpec{},
+						CreatePipeline: &pipelines.CreatePipeline{},
 					},
 				},
 			},
@@ -107,7 +107,7 @@ func TestLookup_NominalWithFilters(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"bar": {
-						PipelineSpec: &pipelines.PipelineSpec{},
+						CreatePipeline: &pipelines.CreatePipeline{},
 					},
 				},
 			},
diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go
index bdcf0f142..1cd6e8743 100644
--- a/bundle/run/pipeline.go
+++ b/bundle/run/pipeline.go
@@ -79,10 +79,10 @@ type pipelineRunner struct {
 }
 
 func (r *pipelineRunner) Name() string {
-	if r.pipeline == nil || r.pipeline.PipelineSpec == nil {
+	if r.pipeline == nil || r.pipeline.CreatePipeline == nil {
 		return ""
 	}
-	return r.pipeline.PipelineSpec.Name
+	return r.pipeline.CreatePipeline.Name
 }
 
 func (r *pipelineRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, error) {
diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json
index 7c72c440e..9d4304cd8 100644
--- a/bundle/schema/jsonschema.json
+++ b/bundle/schema/jsonschema.json
@@ -703,6 +703,9 @@
                         "description": "Restart window of this pipeline.",
                         "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.RestartWindow"
                       },
+                      "run_as": {
+                        "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.RunAs"
+                      },
                       "schema": {
                         "description": "The default schema (database) where tables are read from or published to. The presence of this field implies that the pipeline is in direct publishing mode.",
                         "$ref": "#/$defs/string"
@@ -5385,6 +5388,29 @@
                 }
               ]
             },
+            "pipelines.RunAs": {
+              "oneOf": [
+                {
+                  "type": "object",
+                  "description": "Write-only setting, available only in Create/Update calls. Specifies the user or service principal that the pipeline runs as. If not specified, the pipeline runs as the user who created the pipeline.\n\nOnly `user_name` or `service_principal_name` can be specified. If both are specified, an error is thrown.",
+                  "properties": {
+                    "service_principal_name": {
+                      "description": "Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.",
+                      "$ref": "#/$defs/string"
+                    },
+                    "user_name": {
+                      "description": "The email of an active workspace user. Users can only set this field to their own email.",
+                      "$ref": "#/$defs/string"
+                    }
+                  },
+                  "additionalProperties": false
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
+              ]
+            },
             "pipelines.SchemaSpec": {
               "oneOf": [
                 {
diff --git a/cmd/bundle/generate/pipeline.go b/cmd/bundle/generate/pipeline.go
index 1d2c345d6..9bf9e9947 100644
--- a/cmd/bundle/generate/pipeline.go
+++ b/cmd/bundle/generate/pipeline.go
@@ -92,7 +92,7 @@ func NewGeneratePipelineCommand() *cobra.Command {
 		}
 
 		saver := yamlsaver.NewSaverWithStyle(
-			// Including all PipelineSpec and nested fields which are map[string]string type
+			// Including all CreatePipeline and nested fields which are map[string]string type
 			map[string]yaml.Style{
 				"spark_conf":    yaml.DoubleQuotedStyle,
 				"custom_tags":   yaml.DoubleQuotedStyle,
diff --git a/libs/dyn/drop_keys.go b/libs/dyn/drop_keys.go
new file mode 100644
index 000000000..494f9b9cd
--- /dev/null
+++ b/libs/dyn/drop_keys.go
@@ -0,0 +1,27 @@
+package dyn
+
+func DropKeys(v Value, drop []string) (Value, error) {
+	var err error
+	nv, err := Walk(v, func(p Path, v Value) (Value, error) {
+		if len(p) == 0 {
+			return v, nil
+		}
+
+		// Check if this key should be dropped.
+		for _, key := range drop {
+			if p[0].Key() != key {
+				continue
+			}
+
+			return InvalidValue, ErrDrop
+		}
+
+		// Pass through all other values.
+		return v, ErrSkip
+	})
+	if err != nil {
+		return InvalidValue, err
+	}
+
+	return nv, nil
+}
diff --git a/libs/dyn/drop_keys_test.go b/libs/dyn/drop_keys_test.go
new file mode 100644
index 000000000..83a9744ca
--- /dev/null
+++ b/libs/dyn/drop_keys_test.go
@@ -0,0 +1,24 @@
+package dyn
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDropKeysTest(t *testing.T) {
+	v := V(map[string]Value{
+		"key1": V("value1"),
+		"key2": V("value2"),
+		"key3": V("value3"),
+	})
+
+	vout, err := DropKeys(v, []string{"key1", "key3"})
+	require.NoError(t, err)
+
+	mv := vout.MustMap()
+	require.Equal(t, 1, mv.Len())
+	v, ok := mv.GetByString("key2")
+	require.True(t, ok)
+	require.Equal(t, "value2", v.MustString())
+}

From 989aabe5f1c4d91aa1f9b611f8b49c8241023744 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 7 Feb 2025 18:42:35 +0100
Subject: [PATCH 212/247] acc: Make variable tests local-only (#2312)

Makes use of #2294
---
 acceptance/bundle/variables/env_overrides/test.toml            | 2 --
 acceptance/bundle/variables/resolve-builtin/test.toml          | 2 --
 .../bundle/variables/resolve-vars-in-root-path/test.toml       | 2 --
 acceptance/bundle/variables/test.toml                          | 3 +++
 4 files changed, 3 insertions(+), 6 deletions(-)
 delete mode 100644 acceptance/bundle/variables/env_overrides/test.toml
 delete mode 100644 acceptance/bundle/variables/resolve-builtin/test.toml
 delete mode 100644 acceptance/bundle/variables/resolve-vars-in-root-path/test.toml
 create mode 100644 acceptance/bundle/variables/test.toml

diff --git a/acceptance/bundle/variables/env_overrides/test.toml b/acceptance/bundle/variables/env_overrides/test.toml
deleted file mode 100644
index 439c2fab1..000000000
--- a/acceptance/bundle/variables/env_overrides/test.toml
+++ /dev/null
@@ -1,2 +0,0 @@
-# Cloud run fails with Error: failed to resolve cluster-policy: wrong-cluster-policy, err: Policy named 'wrong-cluster-policy' does not exist
-LocalOnly = true
diff --git a/acceptance/bundle/variables/resolve-builtin/test.toml b/acceptance/bundle/variables/resolve-builtin/test.toml
deleted file mode 100644
index 085fab6c0..000000000
--- a/acceptance/bundle/variables/resolve-builtin/test.toml
+++ /dev/null
@@ -1,2 +0,0 @@
-# Cloud run fails with Error: Path (TestResolveVariableReferences/bar/baz) doesn't start with '/'
-LocalOnly = true
diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/test.toml b/acceptance/bundle/variables/resolve-vars-in-root-path/test.toml
deleted file mode 100644
index d833bd848..000000000
--- a/acceptance/bundle/variables/resolve-vars-in-root-path/test.toml
+++ /dev/null
@@ -1,2 +0,0 @@
-# Cloud run fails with Error: Path (TestResolveVariableReferencesToBundleVariables/bar/files) doesn't start with '/'
-LocalOnly = true
diff --git a/acceptance/bundle/variables/test.toml b/acceptance/bundle/variables/test.toml
new file mode 100644
index 000000000..32398e828
--- /dev/null
+++ b/acceptance/bundle/variables/test.toml
@@ -0,0 +1,3 @@
+# The tests here intend to test variable interpolation via "bundle validate".
+# Even though "bundle validate" does a few API calls, that's not the focus there.
+LocalOnly = true

From 6d83ffd1090fc3b80f9b9957aa1402ea47cfb9ef Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 7 Feb 2025 18:42:47 +0100
Subject: [PATCH 213/247] acc: enable bundle/scripts on cloud (#2313)

---
 acceptance/bundle/scripts/test.toml | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 acceptance/bundle/scripts/test.toml

diff --git a/acceptance/bundle/scripts/test.toml b/acceptance/bundle/scripts/test.toml
deleted file mode 100644
index 1dbd78681..000000000
--- a/acceptance/bundle/scripts/test.toml
+++ /dev/null
@@ -1 +0,0 @@
-LocalOnly = true  # Deployment currently fails when run locally; once that is fixed, remove this setting

From 2a97dcaa45d14981610619f334428b1a6858c332 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Fri, 7 Feb 2025 17:55:16 +0000
Subject: [PATCH 214/247] Raise an error when there are multiple local
 libraries with the same basename used (#2297)

## Changes
Raise an error when there are multiple local libraries with the same
basename used

Fixes #1674

## Tests
Added an unit test
---
 .../same_name_libraries/databricks.yml        |  50 ++++++++
 .../artifacts/same_name_libraries/output.txt  |  14 ++
 .../artifacts/same_name_libraries/script      |   2 +
 .../artifacts/same_name_libraries/test.toml   |   0
 .../same_name_libraries/whl1/setup.py         |  36 ++++++
 .../whl1/src/my_default_python/__init__.py    |   1 +
 .../whl1/src/my_default_python/main.py        |   1 +
 .../same_name_libraries/whl2/setup.py         |  36 ++++++
 .../whl2/src/my_default_python/__init__.py    |   1 +
 .../whl2/src/my_default_python/main.py        |   1 +
 bundle/libraries/expand_glob_references.go    |   2 +-
 bundle/libraries/same_name_libraries.go       |  97 ++++++++++++++
 bundle/libraries/same_name_libraries_test.go  | 121 ++++++++++++++++++
 bundle/phases/deploy.go                       |   5 +
 14 files changed, 366 insertions(+), 1 deletion(-)
 create mode 100644 acceptance/bundle/artifacts/same_name_libraries/databricks.yml
 create mode 100644 acceptance/bundle/artifacts/same_name_libraries/output.txt
 create mode 100644 acceptance/bundle/artifacts/same_name_libraries/script
 create mode 100644 acceptance/bundle/artifacts/same_name_libraries/test.toml
 create mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py
 create mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py
 create mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py
 create mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py
 create mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py
 create mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py
 create mode 100644 bundle/libraries/same_name_libraries.go
 create mode 100644 bundle/libraries/same_name_libraries_test.go

diff --git a/acceptance/bundle/artifacts/same_name_libraries/databricks.yml b/acceptance/bundle/artifacts/same_name_libraries/databricks.yml
new file mode 100644
index 000000000..a065bae76
--- /dev/null
+++ b/acceptance/bundle/artifacts/same_name_libraries/databricks.yml
@@ -0,0 +1,50 @@
+bundle:
+  name: same_name_libraries
+
+variables:
+  cluster:
+    default:
+      spark_version: 15.4.x-scala2.12
+      node_type_id: i3.xlarge
+      data_security_mode: SINGLE_USER
+      num_workers: 0
+      spark_conf:
+          spark.master: "local[*, 4]"
+          spark.databricks.cluster.profile: singleNode
+      custom_tags:
+        ResourceClass: SingleNode
+
+artifacts:
+  whl1:
+    type: whl
+    path: ./whl1
+  whl2:
+    type: whl
+    path: ./whl2
+
+resources:
+  jobs:
+    test:
+      name: "test"
+      tasks:
+        - task_key: task1
+          new_cluster: ${var.cluster}
+          python_wheel_task:
+            entry_point: main
+            package_name: my_default_python
+          libraries:
+            - whl: ./whl1/dist/*.whl
+        - task_key: task2
+          new_cluster: ${var.cluster}
+          python_wheel_task:
+            entry_point: main
+            package_name: my_default_python
+          libraries:
+            - whl: ./whl2/dist/*.whl
+        - task_key: task3
+          new_cluster: ${var.cluster}
+          python_wheel_task:
+            entry_point: main
+            package_name: my_default_python
+          libraries:
+            - whl: ./whl1/dist/*.whl
diff --git a/acceptance/bundle/artifacts/same_name_libraries/output.txt b/acceptance/bundle/artifacts/same_name_libraries/output.txt
new file mode 100644
index 000000000..38cdd43c4
--- /dev/null
+++ b/acceptance/bundle/artifacts/same_name_libraries/output.txt
@@ -0,0 +1,14 @@
+
+>>> errcode [CLI] bundle deploy
+Building whl1...
+Building whl2...
+Error: Duplicate local library name my_default_python-0.0.1-py3-none-any.whl
+  at resources.jobs.test.tasks[0].libraries[0].whl
+     resources.jobs.test.tasks[1].libraries[0].whl
+  in databricks.yml:36:15
+     databricks.yml:43:15
+
+Local library names must be unique
+
+
+Exit code: 1
diff --git a/acceptance/bundle/artifacts/same_name_libraries/script b/acceptance/bundle/artifacts/same_name_libraries/script
new file mode 100644
index 000000000..6c899df07
--- /dev/null
+++ b/acceptance/bundle/artifacts/same_name_libraries/script
@@ -0,0 +1,2 @@
+trace errcode $CLI bundle deploy
+rm -rf whl1 whl2
diff --git a/acceptance/bundle/artifacts/same_name_libraries/test.toml b/acceptance/bundle/artifacts/same_name_libraries/test.toml
new file mode 100644
index 000000000..e69de29bb
diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py
new file mode 100644
index 000000000..1afaf3a4f
--- /dev/null
+++ b/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py
@@ -0,0 +1,36 @@
+"""
+setup.py configuration script describing how to build and package this project.
+
+This file is primarily used by the setuptools library and typically should not
+be executed directly. See README.md for how to deploy, test, and run
+the my_default_python project.
+"""
+
+from setuptools import setup, find_packages
+
+import sys
+
+sys.path.append("./src")
+
+import my_default_python
+
+setup(
+    name="my_default_python",
+    version=my_default_python.__version__,
+    url="https://databricks.com",
+    author="[USERNAME]",
+    description="wheel file based on my_default_python/src",
+    packages=find_packages(where="./src"),
+    package_dir={"": "src"},
+    entry_points={
+        "packages": [
+            "main=my_default_python.main:main",
+        ],
+    },
+    install_requires=[
+        # Dependencies in case the output wheel file is used as a library dependency.
+        # For defining dependencies, when this package is used in Databricks, see:
+        # https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
+        "setuptools"
+    ],
+)
diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py
new file mode 100644
index 000000000..f102a9cad
--- /dev/null
+++ b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py
@@ -0,0 +1 @@
+__version__ = "0.0.1"
diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py
new file mode 100644
index 000000000..11b15b1a4
--- /dev/null
+++ b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py
@@ -0,0 +1 @@
+print("hello")
diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py
new file mode 100644
index 000000000..1afaf3a4f
--- /dev/null
+++ b/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py
@@ -0,0 +1,36 @@
+"""
+setup.py configuration script describing how to build and package this project.
+
+This file is primarily used by the setuptools library and typically should not
+be executed directly. See README.md for how to deploy, test, and run
+the my_default_python project.
+"""
+
+from setuptools import setup, find_packages
+
+import sys
+
+sys.path.append("./src")
+
+import my_default_python
+
+setup(
+    name="my_default_python",
+    version=my_default_python.__version__,
+    url="https://databricks.com",
+    author="[USERNAME]",
+    description="wheel file based on my_default_python/src",
+    packages=find_packages(where="./src"),
+    package_dir={"": "src"},
+    entry_points={
+        "packages": [
+            "main=my_default_python.main:main",
+        ],
+    },
+    install_requires=[
+        # Dependencies in case the output wheel file is used as a library dependency.
+        # For defining dependencies, when this package is used in Databricks, see:
+        # https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
+        "setuptools"
+    ],
+)
diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py
new file mode 100644
index 000000000..f102a9cad
--- /dev/null
+++ b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py
@@ -0,0 +1 @@
+__version__ = "0.0.1"
diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py
new file mode 100644
index 000000000..11b15b1a4
--- /dev/null
+++ b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py
@@ -0,0 +1 @@
+print("hello")
diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go
index bb1905045..7a808f627 100644
--- a/bundle/libraries/expand_glob_references.go
+++ b/bundle/libraries/expand_glob_references.go
@@ -92,7 +92,7 @@ func expandLibraries(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostic
 
 		for _, match := range matches {
 			output = append(output, dyn.NewValue(map[string]dyn.Value{
-				libType: dyn.V(match),
+				libType: dyn.NewValue(match, lib.Locations()),
 			}, lib.Locations()))
 		}
 	}
diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go
new file mode 100644
index 000000000..88b96ab54
--- /dev/null
+++ b/bundle/libraries/same_name_libraries.go
@@ -0,0 +1,97 @@
+package libraries
+
+import (
+	"context"
+	"path/filepath"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+)
+
+type checkForSameNameLibraries struct{}
+
+var patterns = []dyn.Pattern{
+	taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.AnyKey()),
+	forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.AnyKey()),
+	envDepsPattern.Append(dyn.AnyIndex()),
+}
+
+type libData struct {
+	fullPath  string
+	locations []dyn.Location
+	paths     []dyn.Path
+}
+
+func (c checkForSameNameLibraries) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	var diags diag.Diagnostics
+	libs := make(map[string]*libData)
+
+	err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
+		var err error
+		for _, pattern := range patterns {
+			v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, lv dyn.Value) (dyn.Value, error) {
+				libPath := lv.MustString()
+				// If not local library, skip the check
+				if !IsLibraryLocal(libPath) {
+					return lv, nil
+				}
+
+				libFullPath := lv.MustString()
+				lib := filepath.Base(libFullPath)
+				// If the same basename was seen already but full path is different
+				// then it's a duplicate. Add the location to the location list.
+				lp, ok := libs[lib]
+				if !ok {
+					libs[lib] = &libData{
+						fullPath:  libFullPath,
+						locations: []dyn.Location{lv.Location()},
+						paths:     []dyn.Path{p},
+					}
+				} else if lp.fullPath != libFullPath {
+					lp.locations = append(lp.locations, lv.Location())
+					lp.paths = append(lp.paths, p)
+				}
+
+				return lv, nil
+			})
+			if err != nil {
+				return dyn.InvalidValue, err
+			}
+		}
+
+		if err != nil {
+			return dyn.InvalidValue, err
+		}
+
+		return v, nil
+	})
+
+	// Iterate over all the libraries and check if there are any duplicates.
+	// Duplicates will have more than one location.
+	// If there are duplicates, add a diagnostic.
+	for lib, lv := range libs {
+		if len(lv.locations) > 1 {
+			diags = append(diags, diag.Diagnostic{
+				Severity:  diag.Error,
+				Summary:   "Duplicate local library name " + lib,
+				Detail:    "Local library names must be unique",
+				Locations: lv.locations,
+				Paths:     lv.paths,
+			})
+		}
+	}
+	if err != nil {
+		diags = diags.Extend(diag.FromErr(err))
+	}
+
+	return diags
+}
+
+func (c checkForSameNameLibraries) Name() string {
+	return "CheckForSameNameLibraries"
+}
+
+func CheckForSameNameLibraries() bundle.Mutator {
+	return checkForSameNameLibraries{}
+}
diff --git a/bundle/libraries/same_name_libraries_test.go b/bundle/libraries/same_name_libraries_test.go
new file mode 100644
index 000000000..42c38773b
--- /dev/null
+++ b/bundle/libraries/same_name_libraries_test.go
@@ -0,0 +1,121 @@
+package libraries
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/internal/bundletest"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/databricks-sdk-go/service/compute"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSameNameLibraries(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"test": {
+						JobSettings: &jobs.JobSettings{
+							Tasks: []jobs.Task{
+								{
+									Libraries: []compute.Library{
+										{
+											Whl: "full/path/test.whl",
+										},
+									},
+								},
+								{
+									Libraries: []compute.Library{
+										{
+											Whl: "other/path/test.whl",
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	bundletest.SetLocation(b, "resources.jobs.test.tasks[0]", []dyn.Location{
+		{File: "databricks.yml", Line: 10, Column: 1},
+	})
+	bundletest.SetLocation(b, "resources.jobs.test.tasks[1]", []dyn.Location{
+		{File: "databricks.yml", Line: 20, Column: 1},
+	})
+
+	diags := bundle.Apply(context.Background(), b, CheckForSameNameLibraries())
+	require.Len(t, diags, 1)
+	require.Equal(t, diag.Error, diags[0].Severity)
+	require.Equal(t, "Duplicate local library name test.whl", diags[0].Summary)
+	require.Equal(t, []dyn.Location{
+		{File: "databricks.yml", Line: 10, Column: 1},
+		{File: "databricks.yml", Line: 20, Column: 1},
+	}, diags[0].Locations)
+
+	paths := make([]string, 0)
+	for _, p := range diags[0].Paths {
+		paths = append(paths, p.String())
+	}
+	require.Equal(t, []string{
+		"resources.jobs.test.tasks[0].libraries[0].whl",
+		"resources.jobs.test.tasks[1].libraries[0].whl",
+	}, paths)
+}
+
+func TestSameNameLibrariesWithUniqueLibraries(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"test": {
+						JobSettings: &jobs.JobSettings{
+							Tasks: []jobs.Task{
+								{
+									Libraries: []compute.Library{
+										{
+											Whl: "full/path/test-0.1.1.whl",
+										},
+
+										{
+											Whl: "cowsay",
+										},
+									},
+								},
+								{
+									Libraries: []compute.Library{
+										{
+											Whl: "other/path/test-0.1.0.whl",
+										},
+
+										{
+											Whl: "cowsay",
+										},
+									},
+								},
+								{
+									Libraries: []compute.Library{
+										{
+											Whl: "full/path/test-0.1.1.whl", // Use the same library as the first task
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	diags := bundle.Apply(context.Background(), b, CheckForSameNameLibraries())
+	require.Empty(t, diags)
+}
diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go
index c6ec04962..2e9211a7e 100644
--- a/bundle/phases/deploy.go
+++ b/bundle/phases/deploy.go
@@ -155,6 +155,11 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator {
 				mutator.ValidateGitDetails(),
 				artifacts.CleanUp(),
 				libraries.ExpandGlobReferences(),
+				// libraries.CheckForSameNameLibraries() needs to be run after we expand glob references so we
+				// know what are the actual library paths.
+				// libraries.ExpandGlobReferences() has to be run after the libraries are built and thus this
+				// mutator is part of the deploy step rather than validate.
+				libraries.CheckForSameNameLibraries(),
 				libraries.Upload(),
 				trampoline.TransformWheelTask(),
 				files.Upload(outputHandler),

From f8aaa7fce337ac95dba354ddb22cc5a8fa5de046 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Fri, 7 Feb 2025 18:37:03 +0000
Subject: [PATCH 215/247] Added support to generate Git based jobs (#2304)

## Changes
This will generate bundle YAML configuration for Git based jobs but
won't download any related files as they are in Git repo.

Fixes #1423

## Tests
Added unit test

---------

Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>
---
 .../bundle/generate/git_job/databricks.yml    |  2 +
 .../bundle/generate/git_job/out.job.yml       | 17 +++++++
 acceptance/bundle/generate/git_job/output.txt |  2 +
 acceptance/bundle/generate/git_job/script     |  1 +
 acceptance/bundle/generate/git_job/test.toml  | 33 +++++++++++++
 bundle/config/generate/job.go                 |  1 -
 cmd/bundle/generate/job.go                    | 20 ++++++--
 libs/dyn/yamlsaver/utils.go                   | 47 +++++++++++++++++-
 libs/dyn/yamlsaver/utils_test.go              | 48 +++++++++++++++++++
 9 files changed, 165 insertions(+), 6 deletions(-)
 create mode 100644 acceptance/bundle/generate/git_job/databricks.yml
 create mode 100644 acceptance/bundle/generate/git_job/out.job.yml
 create mode 100644 acceptance/bundle/generate/git_job/output.txt
 create mode 100644 acceptance/bundle/generate/git_job/script
 create mode 100644 acceptance/bundle/generate/git_job/test.toml

diff --git a/acceptance/bundle/generate/git_job/databricks.yml b/acceptance/bundle/generate/git_job/databricks.yml
new file mode 100644
index 000000000..adaa7aab3
--- /dev/null
+++ b/acceptance/bundle/generate/git_job/databricks.yml
@@ -0,0 +1,2 @@
+bundle:
+  name: git_job
diff --git a/acceptance/bundle/generate/git_job/out.job.yml b/acceptance/bundle/generate/git_job/out.job.yml
new file mode 100644
index 000000000..0eb2a3fb1
--- /dev/null
+++ b/acceptance/bundle/generate/git_job/out.job.yml
@@ -0,0 +1,17 @@
+resources:
+  jobs:
+    out:
+      name: gitjob
+      tasks:
+        - task_key: test_task
+          notebook_task:
+            notebook_path: some/test/notebook.py
+        - task_key: test_task_2
+          notebook_task:
+            notebook_path: /Workspace/Users/foo@bar.com/some/test/notebook.py
+            source: WORKSPACE
+      git_source:
+        git_branch: main
+        git_commit: abcdef
+        git_provider: github
+        git_url: https://git.databricks.com
diff --git a/acceptance/bundle/generate/git_job/output.txt b/acceptance/bundle/generate/git_job/output.txt
new file mode 100644
index 000000000..680c92ff9
--- /dev/null
+++ b/acceptance/bundle/generate/git_job/output.txt
@@ -0,0 +1,2 @@
+Job is using Git source, skipping downloading files
+Job configuration successfully saved to out.job.yml
diff --git a/acceptance/bundle/generate/git_job/script b/acceptance/bundle/generate/git_job/script
new file mode 100644
index 000000000..7598966b0
--- /dev/null
+++ b/acceptance/bundle/generate/git_job/script
@@ -0,0 +1 @@
+$CLI bundle generate job --existing-job-id 1234 --config-dir . --key out
diff --git a/acceptance/bundle/generate/git_job/test.toml b/acceptance/bundle/generate/git_job/test.toml
new file mode 100644
index 000000000..28b473245
--- /dev/null
+++ b/acceptance/bundle/generate/git_job/test.toml
@@ -0,0 +1,33 @@
+LocalOnly = true  # This test needs to run against stubbed Databricks API
+
+[[Server]]
+Pattern = "GET /api/2.1/jobs/get"
+Response.Body = '''
+{
+    "job_id": 11223344,
+    "settings": {
+        "name": "gitjob",
+        "git_source": {
+            "git_url":      "https://git.databricks.com",
+            "git_provider": "github",
+            "git_branch":   "main",
+            "git_commit":   "abcdef"
+        },
+        "tasks": [
+            {
+                "task_key": "test_task",
+                "notebook_task": {
+                    "notebook_path": "some/test/notebook.py"
+                }
+            },
+            {
+                "task_key": "test_task_2",
+                "notebook_task": {
+                    "source": "WORKSPACE",
+                    "notebook_path": "/Workspace/Users/foo@bar.com/some/test/notebook.py"
+                }
+            }
+        ]
+    }
+}
+'''
diff --git a/bundle/config/generate/job.go b/bundle/config/generate/job.go
index 0cdcbf3ad..934eda2cf 100644
--- a/bundle/config/generate/job.go
+++ b/bundle/config/generate/job.go
@@ -13,7 +13,6 @@ var (
 
 func ConvertJobToValue(job *jobs.Job) (dyn.Value, error) {
 	value := make(map[string]dyn.Value)
-
 	if job.Settings.Tasks != nil {
 		tasks := make([]dyn.Value, 0)
 		for _, task := range job.Settings.Tasks {
diff --git a/cmd/bundle/generate/job.go b/cmd/bundle/generate/job.go
index d97891cd5..438b235c9 100644
--- a/cmd/bundle/generate/job.go
+++ b/cmd/bundle/generate/job.go
@@ -50,10 +50,22 @@ func NewGenerateJobCommand() *cobra.Command {
 		}
 
 		downloader := newDownloader(w, sourceDir, configDir)
-		for _, task := range job.Settings.Tasks {
-			err := downloader.MarkTaskForDownload(ctx, &task)
-			if err != nil {
-				return err
+
+		// Don't download files if the job is using Git source
+		// When Git source is used, the job will be using the files from the Git repository
+		// but specific tasks might override this behaviour by using `source: WORKSPACE` setting.
+		// In this case, we don't want to download the files as well for these specific tasks
+		// because it leads to confusion with relative paths between workspace and GIT files.
+		// Instead we keep these tasks as is and let the user handle the files manually.
+		// The configuration will be deployable as tasks paths for source: WORKSPACE tasks will be absolute workspace paths.
+		if job.Settings.GitSource != nil {
+			cmdio.LogString(ctx, "Job is using Git source, skipping downloading files")
+		} else {
+			for _, task := range job.Settings.Tasks {
+				err := downloader.MarkTaskForDownload(ctx, &task)
+				if err != nil {
+					return err
+				}
 			}
 		}
 
diff --git a/libs/dyn/yamlsaver/utils.go b/libs/dyn/yamlsaver/utils.go
index a162bf31f..c1b60b1b5 100644
--- a/libs/dyn/yamlsaver/utils.go
+++ b/libs/dyn/yamlsaver/utils.go
@@ -22,9 +22,50 @@ func ConvertToMapValue(strct any, order *Order, skipFields []string, dst map[str
 		return dyn.InvalidValue, fmt.Errorf("expected map, got %s", mv.Kind())
 	}
 
+	mv, err = sortMapAlphabetically(mv)
+	if err != nil {
+		return dyn.InvalidValue, err
+	}
+
 	return skipAndOrder(mv, order, skipFields, dst)
 }
 
+// Sort the map alphabetically by keys. This is used to produce stable output for generated YAML files.
+func sortMapAlphabetically(mv dyn.Value) (dyn.Value, error) {
+	sortedMap := dyn.NewMapping()
+	mapV := mv.MustMap()
+	keys := mapV.Keys()
+	slices.SortStableFunc(keys, func(i, j dyn.Value) int {
+		iKey := i.MustString()
+		jKey := j.MustString()
+		if iKey < jKey {
+			return -1
+		}
+
+		if iKey > jKey {
+			return 1
+		}
+		return 0
+	})
+
+	for _, key := range keys {
+		value, _ := mapV.Get(key)
+		var err error
+		if value.Kind() == dyn.KindMap {
+			value, err = sortMapAlphabetically(value)
+			if err != nil {
+				return dyn.InvalidValue, err
+			}
+		}
+		err = sortedMap.Set(key, value)
+		if err != nil {
+			return dyn.InvalidValue, err
+		}
+	}
+
+	return dyn.V(sortedMap), nil
+}
+
 func skipAndOrder(mv dyn.Value, order *Order, skipFields []string, dst map[string]dyn.Value) (dyn.Value, error) {
 	for _, pair := range mv.MustMap().Pairs() {
 		k := pair.Key.MustString()
@@ -44,7 +85,11 @@ func skipAndOrder(mv dyn.Value, order *Order, skipFields []string, dst map[strin
 			continue
 		}
 
-		dst[k] = dyn.NewValue(v.Value(), []dyn.Location{{Line: order.Get(k)}})
+		if order == nil {
+			dst[k] = v
+		} else {
+			dst[k] = dyn.NewValue(v.Value(), []dyn.Location{{Line: order.Get(k)}})
+		}
 	}
 
 	return dyn.V(dst), nil
diff --git a/libs/dyn/yamlsaver/utils_test.go b/libs/dyn/yamlsaver/utils_test.go
index 1afab601a..f7ea3c96c 100644
--- a/libs/dyn/yamlsaver/utils_test.go
+++ b/libs/dyn/yamlsaver/utils_test.go
@@ -7,6 +7,54 @@ import (
 	assert "github.com/databricks/cli/libs/dyn/dynassert"
 )
 
+func TestConvertToMap(t *testing.T) {
+	type test struct {
+		Name            string            `json:"name"`
+		Map             map[string]string `json:"map"`
+		List            []string          `json:"list"`
+		LongNameField   string            `json:"long_name_field"`
+		ForceSendFields []string          `json:"-"`
+		Format          string            `json:"format"`
+	}
+
+	v := &test{
+		Name: "test",
+		Map: map[string]string{
+			"key2": "value2",
+			"key1": "value1",
+		},
+		List: []string{"a", "b", "c"},
+		ForceSendFields: []string{
+			"Name",
+		},
+		LongNameField: "long name goes here",
+	}
+	result, err := ConvertToMapValue(v, nil, []string{"format"}, map[string]dyn.Value{})
+	assert.NoError(t, err)
+	assert.Equal(t, dyn.V(map[string]dyn.Value{
+		"list": dyn.NewValue(
+			[]dyn.Value{
+				dyn.V("a"),
+				dyn.V("b"),
+				dyn.V("c"),
+			},
+			[]dyn.Location{},
+		),
+		"long_name_field": dyn.NewValue("long name goes here", []dyn.Location{}),
+		"map": dyn.NewValue(
+			map[string]dyn.Value{
+				"key1": dyn.V("value1"),
+				"key2": dyn.V("value2"),
+			},
+			[]dyn.Location{},
+		),
+		"name": dyn.NewValue(
+			"test",
+			[]dyn.Location{},
+		),
+	}), result)
+}
+
 func TestConvertToMapValueWithOrder(t *testing.T) {
 	type test struct {
 		Name            string            `json:"name"`

From 06e342afc58139264fabd34eec603d58f2b7f95a Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 10 Feb 2025 10:16:31 +0100
Subject: [PATCH 216/247] Silence a comment in Makefile (#2315)

It was not indended to be printed. Follow up to #2298
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 7da7e4789..fb3936184 100644
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@ lint:
 	golangci-lint run --fix
 
 tidy:
-	# not part of golangci-lint, apparently
+	@# not part of golangci-lint, apparently
 	go mod tidy
 
 lintcheck:

From 2175dd24a4935963eea8ad1c95b72d309d4ea780 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 10 Feb 2025 11:42:39 +0100
Subject: [PATCH 217/247] Do not gitignore .databricks and terraform (#2318)

For acceptance/bundle/templates I'd like to run "bundle deploy". This
would create .databricks directory inside materialized output. It might
makes sense to commit some of this as part of golden files output. Even
if we did not commit anything, the test runner will see those files and
show the difference. Thus git should also see them.

Also rename .gitignore to out.gitignore in those tests, since that
includes .databricks as well.
---
 .gitignore                                                    | 4 ----
 .../dbt-sql/output/my_dbt_sql/{.gitignore => out.gitignore}   | 0
 acceptance/bundle/templates/dbt-sql/script                    | 3 +++
 .../output/my_default_python/{.gitignore => out.gitignore}    | 0
 acceptance/bundle/templates/default-python/script             | 3 +++
 .../output/my_default_sql/{.gitignore => out.gitignore}       | 0
 acceptance/bundle/templates/default-sql/script                | 3 +++
 .../output/my_jobs_as_code/{.gitignore => out.gitignore}      | 0
 acceptance/bundle/templates/experimental-jobs-as-code/script  | 3 +++
 9 files changed, 12 insertions(+), 4 deletions(-)
 rename acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/{.gitignore => out.gitignore} (100%)
 rename acceptance/bundle/templates/default-python/output/my_default_python/{.gitignore => out.gitignore} (100%)
 rename acceptance/bundle/templates/default-sql/output/my_default_sql/{.gitignore => out.gitignore} (100%)
 rename acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/{.gitignore => out.gitignore} (100%)

diff --git a/.gitignore b/.gitignore
index 2060b6bac..35aef1764 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,11 +25,7 @@ coverage-acceptance.txt
 __pycache__
 *.pyc
 
-.terraform
-.terraform.lock.hcl
-
 .vscode/launch.json
 .vscode/tasks.json
 
-.databricks
 .ruff_cache
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.gitignore b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/out.gitignore
similarity index 100%
rename from acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.gitignore
rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/out.gitignore
diff --git a/acceptance/bundle/templates/dbt-sql/script b/acceptance/bundle/templates/dbt-sql/script
index c4ca817fe..3a2660de5 100644
--- a/acceptance/bundle/templates/dbt-sql/script
+++ b/acceptance/bundle/templates/dbt-sql/script
@@ -3,3 +3,6 @@ trace $CLI bundle init dbt-sql --config-file ./input.json --output-dir output
 cd output/my_dbt_sql
 trace $CLI bundle validate -t dev
 trace $CLI bundle validate -t prod
+
+# Do not affect this repository's git behaviour #2318
+mv .gitignore out.gitignore
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/.gitignore b/acceptance/bundle/templates/default-python/output/my_default_python/out.gitignore
similarity index 100%
rename from acceptance/bundle/templates/default-python/output/my_default_python/.gitignore
rename to acceptance/bundle/templates/default-python/output/my_default_python/out.gitignore
diff --git a/acceptance/bundle/templates/default-python/script b/acceptance/bundle/templates/default-python/script
index b11a7ea21..e5fcb7741 100644
--- a/acceptance/bundle/templates/default-python/script
+++ b/acceptance/bundle/templates/default-python/script
@@ -3,3 +3,6 @@ trace $CLI bundle init default-python --config-file ./input.json --output-dir ou
 cd output/my_default_python
 trace $CLI bundle validate -t dev
 trace $CLI bundle validate -t prod
+
+# Do not affect this repository's git behaviour #2318
+mv .gitignore out.gitignore
diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/.gitignore b/acceptance/bundle/templates/default-sql/output/my_default_sql/out.gitignore
similarity index 100%
rename from acceptance/bundle/templates/default-sql/output/my_default_sql/.gitignore
rename to acceptance/bundle/templates/default-sql/output/my_default_sql/out.gitignore
diff --git a/acceptance/bundle/templates/default-sql/script b/acceptance/bundle/templates/default-sql/script
index 66e7a14a2..7ea0d863c 100644
--- a/acceptance/bundle/templates/default-sql/script
+++ b/acceptance/bundle/templates/default-sql/script
@@ -3,3 +3,6 @@ trace $CLI bundle init default-sql --config-file ./input.json --output-dir outpu
 cd output/my_default_sql
 trace $CLI bundle validate -t dev
 trace $CLI bundle validate -t prod
+
+# Do not affect this repository's git behaviour #2318
+mv .gitignore out.gitignore
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/.gitignore b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/out.gitignore
similarity index 100%
rename from acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/.gitignore
rename to acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/out.gitignore
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/script b/acceptance/bundle/templates/experimental-jobs-as-code/script
index 10188aabd..08e48fc5f 100644
--- a/acceptance/bundle/templates/experimental-jobs-as-code/script
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/script
@@ -8,3 +8,6 @@ uv sync -q
 trace $CLI bundle validate -t dev --output json | jq ".resources"
 
 rm -fr .venv resources/__pycache__ uv.lock my_jobs_as_code.egg-info
+
+# Do not affect this repository's git behaviour #2318
+mv .gitignore out.gitignore

From cc073801855d0e0a27f32b25f2a89c1fa391dc8d Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 10 Feb 2025 11:53:00 +0100
Subject: [PATCH 218/247] acc: Summarize unexpected files (#2320)

## Changes
When there are many unexpected files, it's good to see them as a list
rather than scattered throughout the output.

<!-- Summary of your changes that are easy to understand -->

## Tests
Manually, example output:
```
    acceptance_test.go:363: Test produced unexpected files:
        output/my_default_sql/.databricks/bundle/dev/sync-snapshots/71c79ded90615dc7.json
        output/my_default_sql/.databricks/bundle/dev/terraform/.terraform/providers/registry.terraform.io/databricks/databricks/1.64.1/darwin_arm64
        output/my_default_sql/.databricks/bundle/dev/terraform/plan
        output/my_default_sql/.databricks/bundle/prod/sync-snapshots/83e677e75259c93b.json
        output/my_default_sql/.databricks/bundle/prod/terraform/.terraform/providers/registry.terraform.io/databricks/databricks/1.64.1/darwin_arm64
```
---
 acceptance/acceptance_test.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 4c4404d55..241ab42be 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -343,6 +343,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 
 	// Make sure there are not unaccounted for new files
 	files := ListDir(t, tmpDir)
+	unexpected := []string{}
 	for _, relPath := range files {
 		if _, ok := inputs[relPath]; ok {
 			continue
@@ -350,13 +351,17 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		if _, ok := outputs[relPath]; ok {
 			continue
 		}
-		t.Errorf("Unexpected output: %s", relPath)
+		unexpected = append(unexpected, relPath)
 		if strings.HasPrefix(relPath, "out") {
 			// We have a new file starting with "out"
 			// Show the contents & support overwrite mode for it:
 			doComparison(t, repls, dir, tmpDir, relPath, &printedRepls)
 		}
 	}
+
+	if len(unexpected) > 0 {
+		t.Error("Test produced unexpected files:\n" + strings.Join(unexpected, "\n"))
+	}
 }
 
 func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool) {

From 4ebc86282fbbaec9293af022a8d9f5b9b0b80a4a Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 10 Feb 2025 11:55:34 +0100
Subject: [PATCH 219/247] acc: Split bundle/templates and
 bundle/templates-machinery (#2317)

The tests in acceptance/bundle/template focus on standard templates. The
plan is to extend them with "bundle deploy" and enable them on the
cloud.

The tests in acceptance/bundle/template-machinery focus on specific
aspects of template implementation. Most of them are expected to remain
local-only.
---
 .../helpers-error/databricks_template_schema.json               | 0
 .../{templates => templates-machinery}/helpers-error/output.txt | 0
 .../{templates => templates-machinery}/helpers-error/script     | 0
 .../helpers-error/template/helpers.txt.tmpl                     | 0
 .../{templates => templates-machinery}/helpers-error/test.toml  | 0
 .../helpers/databricks_template_schema.json                     | 0
 .../{templates => templates-machinery}/helpers/output.txt       | 0
 .../bundle/{templates => templates-machinery}/helpers/script    | 0
 .../helpers/template/helpers.txt.tmpl                           | 0
 .../bundle/{templates => templates-machinery}/helpers/test.toml | 0
 acceptance/bundle/templates-machinery/test.toml                 | 2 ++
 .../{templates => templates-machinery}/wrong-path/output.txt    | 0
 .../bundle/{templates => templates-machinery}/wrong-path/script | 0
 .../{templates => templates-machinery}/wrong-path/test.toml     | 0
 .../{templates => templates-machinery}/wrong-url/output.txt     | 0
 .../bundle/{templates => templates-machinery}/wrong-url/script  | 0
 .../{templates => templates-machinery}/wrong-url/test.toml      | 0
 17 files changed, 2 insertions(+)
 rename acceptance/bundle/{templates => templates-machinery}/helpers-error/databricks_template_schema.json (100%)
 rename acceptance/bundle/{templates => templates-machinery}/helpers-error/output.txt (100%)
 rename acceptance/bundle/{templates => templates-machinery}/helpers-error/script (100%)
 rename acceptance/bundle/{templates => templates-machinery}/helpers-error/template/helpers.txt.tmpl (100%)
 rename acceptance/bundle/{templates => templates-machinery}/helpers-error/test.toml (100%)
 rename acceptance/bundle/{templates => templates-machinery}/helpers/databricks_template_schema.json (100%)
 rename acceptance/bundle/{templates => templates-machinery}/helpers/output.txt (100%)
 rename acceptance/bundle/{templates => templates-machinery}/helpers/script (100%)
 rename acceptance/bundle/{templates => templates-machinery}/helpers/template/helpers.txt.tmpl (100%)
 rename acceptance/bundle/{templates => templates-machinery}/helpers/test.toml (100%)
 create mode 100644 acceptance/bundle/templates-machinery/test.toml
 rename acceptance/bundle/{templates => templates-machinery}/wrong-path/output.txt (100%)
 rename acceptance/bundle/{templates => templates-machinery}/wrong-path/script (100%)
 rename acceptance/bundle/{templates => templates-machinery}/wrong-path/test.toml (100%)
 rename acceptance/bundle/{templates => templates-machinery}/wrong-url/output.txt (100%)
 rename acceptance/bundle/{templates => templates-machinery}/wrong-url/script (100%)
 rename acceptance/bundle/{templates => templates-machinery}/wrong-url/test.toml (100%)

diff --git a/acceptance/bundle/templates/helpers-error/databricks_template_schema.json b/acceptance/bundle/templates-machinery/helpers-error/databricks_template_schema.json
similarity index 100%
rename from acceptance/bundle/templates/helpers-error/databricks_template_schema.json
rename to acceptance/bundle/templates-machinery/helpers-error/databricks_template_schema.json
diff --git a/acceptance/bundle/templates/helpers-error/output.txt b/acceptance/bundle/templates-machinery/helpers-error/output.txt
similarity index 100%
rename from acceptance/bundle/templates/helpers-error/output.txt
rename to acceptance/bundle/templates-machinery/helpers-error/output.txt
diff --git a/acceptance/bundle/templates/helpers-error/script b/acceptance/bundle/templates-machinery/helpers-error/script
similarity index 100%
rename from acceptance/bundle/templates/helpers-error/script
rename to acceptance/bundle/templates-machinery/helpers-error/script
diff --git a/acceptance/bundle/templates/helpers-error/template/helpers.txt.tmpl b/acceptance/bundle/templates-machinery/helpers-error/template/helpers.txt.tmpl
similarity index 100%
rename from acceptance/bundle/templates/helpers-error/template/helpers.txt.tmpl
rename to acceptance/bundle/templates-machinery/helpers-error/template/helpers.txt.tmpl
diff --git a/acceptance/bundle/templates/helpers-error/test.toml b/acceptance/bundle/templates-machinery/helpers-error/test.toml
similarity index 100%
rename from acceptance/bundle/templates/helpers-error/test.toml
rename to acceptance/bundle/templates-machinery/helpers-error/test.toml
diff --git a/acceptance/bundle/templates/helpers/databricks_template_schema.json b/acceptance/bundle/templates-machinery/helpers/databricks_template_schema.json
similarity index 100%
rename from acceptance/bundle/templates/helpers/databricks_template_schema.json
rename to acceptance/bundle/templates-machinery/helpers/databricks_template_schema.json
diff --git a/acceptance/bundle/templates/helpers/output.txt b/acceptance/bundle/templates-machinery/helpers/output.txt
similarity index 100%
rename from acceptance/bundle/templates/helpers/output.txt
rename to acceptance/bundle/templates-machinery/helpers/output.txt
diff --git a/acceptance/bundle/templates/helpers/script b/acceptance/bundle/templates-machinery/helpers/script
similarity index 100%
rename from acceptance/bundle/templates/helpers/script
rename to acceptance/bundle/templates-machinery/helpers/script
diff --git a/acceptance/bundle/templates/helpers/template/helpers.txt.tmpl b/acceptance/bundle/templates-machinery/helpers/template/helpers.txt.tmpl
similarity index 100%
rename from acceptance/bundle/templates/helpers/template/helpers.txt.tmpl
rename to acceptance/bundle/templates-machinery/helpers/template/helpers.txt.tmpl
diff --git a/acceptance/bundle/templates/helpers/test.toml b/acceptance/bundle/templates-machinery/helpers/test.toml
similarity index 100%
rename from acceptance/bundle/templates/helpers/test.toml
rename to acceptance/bundle/templates-machinery/helpers/test.toml
diff --git a/acceptance/bundle/templates-machinery/test.toml b/acceptance/bundle/templates-machinery/test.toml
new file mode 100644
index 000000000..9083ecd1b
--- /dev/null
+++ b/acceptance/bundle/templates-machinery/test.toml
@@ -0,0 +1,2 @@
+# Testing template machinery, by default there is no need to check against cloud.
+LocalOnly = true
diff --git a/acceptance/bundle/templates/wrong-path/output.txt b/acceptance/bundle/templates-machinery/wrong-path/output.txt
similarity index 100%
rename from acceptance/bundle/templates/wrong-path/output.txt
rename to acceptance/bundle/templates-machinery/wrong-path/output.txt
diff --git a/acceptance/bundle/templates/wrong-path/script b/acceptance/bundle/templates-machinery/wrong-path/script
similarity index 100%
rename from acceptance/bundle/templates/wrong-path/script
rename to acceptance/bundle/templates-machinery/wrong-path/script
diff --git a/acceptance/bundle/templates/wrong-path/test.toml b/acceptance/bundle/templates-machinery/wrong-path/test.toml
similarity index 100%
rename from acceptance/bundle/templates/wrong-path/test.toml
rename to acceptance/bundle/templates-machinery/wrong-path/test.toml
diff --git a/acceptance/bundle/templates/wrong-url/output.txt b/acceptance/bundle/templates-machinery/wrong-url/output.txt
similarity index 100%
rename from acceptance/bundle/templates/wrong-url/output.txt
rename to acceptance/bundle/templates-machinery/wrong-url/output.txt
diff --git a/acceptance/bundle/templates/wrong-url/script b/acceptance/bundle/templates-machinery/wrong-url/script
similarity index 100%
rename from acceptance/bundle/templates/wrong-url/script
rename to acceptance/bundle/templates-machinery/wrong-url/script
diff --git a/acceptance/bundle/templates/wrong-url/test.toml b/acceptance/bundle/templates-machinery/wrong-url/test.toml
similarity index 100%
rename from acceptance/bundle/templates/wrong-url/test.toml
rename to acceptance/bundle/templates-machinery/wrong-url/test.toml

From ee440e65fec623fbf3e4cba05ac21d67ee6306db Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Mon, 10 Feb 2025 17:48:05 +0530
Subject: [PATCH 220/247] Serialize all header values in acceptance tests
 (#2311)

## Changes
Based on feedback in
https://github.com/databricks/cli/pull/2296#discussion_r1946660650.
Previously we only serialized the first value for a header in the
requests log. Now we serialise all values for a header key.

## Tests
Existing test
---
 .../workspace/jobs/create/out.requests.txt       |  2 +-
 libs/testserver/server.go                        | 16 +++++++++-------
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/acceptance/workspace/jobs/create/out.requests.txt b/acceptance/workspace/jobs/create/out.requests.txt
index 60977e3e3..2510762db 100644
--- a/acceptance/workspace/jobs/create/out.requests.txt
+++ b/acceptance/workspace/jobs/create/out.requests.txt
@@ -1 +1 @@
-{"headers":{"Authorization":"Bearer [DATABRICKS_TOKEN]","User-Agent":"cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}}
+{"headers":{"Authorization":["Bearer [DATABRICKS_TOKEN]"],"User-Agent":["cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"]},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}}
diff --git a/libs/testserver/server.go b/libs/testserver/server.go
index 9ccf34be0..d0c340c12 100644
--- a/libs/testserver/server.go
+++ b/libs/testserver/server.go
@@ -31,10 +31,10 @@ type Server struct {
 }
 
 type Request struct {
-	Headers map[string]string `json:"headers,omitempty"`
-	Method  string            `json:"method"`
-	Path    string            `json:"path"`
-	Body    any               `json:"body"`
+	Headers http.Header `json:"headers,omitempty"`
+	Method  string      `json:"method"`
+	Path    string      `json:"path"`
+	Body    any         `json:"body"`
 }
 
 func New(t testutil.TestingT) *Server {
@@ -109,12 +109,14 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) {
 			body, err := io.ReadAll(r.Body)
 			assert.NoError(s.t, err)
 
-			headers := make(map[string]string)
+			headers := make(http.Header)
 			for k, v := range r.Header {
-				if len(v) == 0 || !slices.Contains(s.IncludeRequestHeaders, k) {
+				if !slices.Contains(s.IncludeRequestHeaders, k) {
 					continue
 				}
-				headers[k] = v[0]
+				for _, vv := range v {
+					headers.Add(k, vv)
+				}
 			}
 
 			s.Requests = append(s.Requests, Request{

From 047691dd9148061610e50aa6c12e04cdb85e4022 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 10 Feb 2025 12:50:52 +0000
Subject: [PATCH 221/247] Bump github.com/databricks/databricks-sdk-go from
 0.56.1 to 0.57.0 (#2321)

Bumps
[github.com/databricks/databricks-sdk-go](https://github.com/databricks/databricks-sdk-go)
from 0.56.1 to 0.57.0.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/databricks/databricks-sdk-go/releases">github.com/databricks/databricks-sdk-go's
releases</a>.</em></p>
<blockquote>
<h2>v0.57.0</h2>
<h2>[Release] Release v0.57.0</h2>
<h3>New Features and Improvements</h3>
<ul>
<li>Add support for async OAuth token refreshes (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1135">#1135</a>).</li>
</ul>
<h3>API Changes:</h3>
<ul>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/billing#BudgetPolicyAPI">a.BudgetPolicy</a>
account-level service.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/settings#EnableIpAccessListsAPI">a.EnableIpAccessLists</a>
account-level service.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/dashboards#LakeviewEmbeddedAPI">w.LakeviewEmbedded</a>
workspace-level service and <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/dashboards#QueryExecutionAPI">w.QueryExecution</a>
workspace-level service.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/sql#RedashConfigAPI">w.RedashConfig</a>
workspace-level service.</li>
<li>Added <code>GcpOauthToken</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#TemporaryCredentials">catalog.TemporaryCredentials</a>.</li>
<li>Added <code>Options</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#UpdateCatalog">catalog.UpdateCatalog</a>.</li>
<li>Added <code>StatementId</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/dashboards#QueryAttachment">dashboards.QueryAttachment</a>.</li>
<li>Added <code>EffectivePerformanceTarget</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#BaseRun">jobs.BaseRun</a>.</li>
<li>Added <code>PerformanceTarget</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#CreateJob">jobs.CreateJob</a>.</li>
<li>Added <code>PerformanceTarget</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#JobSettings">jobs.JobSettings</a>.</li>
<li>Added <code>EffectivePerformanceTarget</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#Run">jobs.Run</a>.</li>
<li>Added <code>PerformanceTarget</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#RunNow">jobs.RunNow</a>.</li>
<li>Added <code>Disabled</code> and
<code>EffectivePerformanceTarget</code> fields for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#RunTask">jobs.RunTask</a>.</li>
<li>Added <code>UserAuthorizedScopes</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/oauth2#CreateCustomAppIntegration">oauth2.CreateCustomAppIntegration</a>.</li>
<li>Added <code>UserAuthorizedScopes</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/oauth2#GetCustomAppIntegrationOutput">oauth2.GetCustomAppIntegrationOutput</a>.</li>
<li>Added <code>UserAuthorizedScopes</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/oauth2#UpdateCustomAppIntegration">oauth2.UpdateCustomAppIntegration</a>.</li>
<li>Added <code>Contents</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#HttpRequestResponse">serving.HttpRequestResponse</a>.</li>
<li>Changed <code>HttpRequest</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service to type <code>HttpRequest</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service.</li>
<li>Changed <code>HttpRequest</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service to return <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#HttpRequestResponse">serving.HttpRequestResponse</a>.</li>
<li>Removed <code>SecurableKind</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#CatalogInfo">catalog.CatalogInfo</a>.</li>
<li>Removed <code>SecurableKind</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#ConnectionInfo">catalog.ConnectionInfo</a>.</li>
<li>Removed <code>StatusCode</code> and <code>Text</code> fields for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ExternalFunctionResponse">serving.ExternalFunctionResponse</a>.</li>
</ul>
<p>OpenAPI SHA: c72c58f97b950fcb924a90ef164bcb10cfcd5ece, Date:
2025-02-03</p>
</blockquote>
</details>
<details>
<summary>Changelog</summary>
<p><em>Sourced from <a
href="https://github.com/databricks/databricks-sdk-go/blob/main/CHANGELOG.md">github.com/databricks/databricks-sdk-go's
changelog</a>.</em></p>
<blockquote>
<h2>[Release] Release v0.57.0</h2>
<h3>New Features and Improvements</h3>
<ul>
<li>Add support for async OAuth token refreshes (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/1135">#1135</a>).</li>
</ul>
<h3>API Changes:</h3>
<ul>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/billing#BudgetPolicyAPI">a.BudgetPolicy</a>
account-level service.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/settings#EnableIpAccessListsAPI">a.EnableIpAccessLists</a>
account-level service.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/dashboards#LakeviewEmbeddedAPI">w.LakeviewEmbedded</a>
workspace-level service and <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/dashboards#QueryExecutionAPI">w.QueryExecution</a>
workspace-level service.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/sql#RedashConfigAPI">w.RedashConfig</a>
workspace-level service.</li>
<li>Added <code>GcpOauthToken</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#TemporaryCredentials">catalog.TemporaryCredentials</a>.</li>
<li>Added <code>Options</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#UpdateCatalog">catalog.UpdateCatalog</a>.</li>
<li>Added <code>StatementId</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/dashboards#QueryAttachment">dashboards.QueryAttachment</a>.</li>
<li>Added <code>EffectivePerformanceTarget</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#BaseRun">jobs.BaseRun</a>.</li>
<li>Added <code>PerformanceTarget</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#CreateJob">jobs.CreateJob</a>.</li>
<li>Added <code>PerformanceTarget</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#JobSettings">jobs.JobSettings</a>.</li>
<li>Added <code>EffectivePerformanceTarget</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#Run">jobs.Run</a>.</li>
<li>Added <code>PerformanceTarget</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#RunNow">jobs.RunNow</a>.</li>
<li>Added <code>Disabled</code> and
<code>EffectivePerformanceTarget</code> fields for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#RunTask">jobs.RunTask</a>.</li>
<li>Added <code>UserAuthorizedScopes</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/oauth2#CreateCustomAppIntegration">oauth2.CreateCustomAppIntegration</a>.</li>
<li>Added <code>UserAuthorizedScopes</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/oauth2#GetCustomAppIntegrationOutput">oauth2.GetCustomAppIntegrationOutput</a>.</li>
<li>Added <code>UserAuthorizedScopes</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/oauth2#UpdateCustomAppIntegration">oauth2.UpdateCustomAppIntegration</a>.</li>
<li>Added <code>Contents</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#HttpRequestResponse">serving.HttpRequestResponse</a>.</li>
<li>Changed <code>HttpRequest</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service to type <code>HttpRequest</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service.</li>
<li>Changed <code>HttpRequest</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service to return <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#HttpRequestResponse">serving.HttpRequestResponse</a>.</li>
<li>Removed <code>SecurableKind</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#CatalogInfo">catalog.CatalogInfo</a>.</li>
<li>Removed <code>SecurableKind</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#ConnectionInfo">catalog.ConnectionInfo</a>.</li>
<li>Removed <code>StatusCode</code> and <code>Text</code> fields for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ExternalFunctionResponse">serving.ExternalFunctionResponse</a>.</li>
</ul>
<p>OpenAPI SHA: c72c58f97b950fcb924a90ef164bcb10cfcd5ece, Date:
2025-02-03</p>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/7cb1883c857f55b1a3ceef90100d3de680eab4e6"><code>7cb1883</code></a>
[Release] Release v0.57.0 (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/1140">#1140</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/31fdc692bb1959ea4be6705526050267e7e6ebf9"><code>31fdc69</code></a>
[Feature] Add support for async OAuth token refreshes (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/1135">#1135</a>)</li>
<li>See full diff in <a
href="https://github.com/databricks/databricks-sdk-go/compare/v0.56.1...v0.57.0">compare
view</a></li>
</ul>
</details>
<br />

<details>
<summary>Most Recent Ignore Conditions Applied to This Pull
Request</summary>

| Dependency Name | Ignore Conditions |
| --- | --- |
| github.com/databricks/databricks-sdk-go | [>= 0.28.a, < 0.29] |
</details>


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/databricks/databricks-sdk-go&package-manager=go_modules&previous-version=0.56.1&new-version=0.57.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Andrew Nester <andrew.nester@databricks.com>
---
 .codegen/_openapi_sha                         |   2 +-
 .gitattributes                                |   5 +
 .../internal/schema/annotations_openapi.yml   |  14 +
 bundle/schema/jsonschema.json                 |  20 +
 cmd/account/budget-policy/budget-policy.go    | 373 ++++++++++++++++++
 cmd/account/cmd.go                            |   2 +
 .../custom-app-integration.go                 |   2 +
 .../enable-ip-access-lists.go                 | 218 ++++++++++
 .../federation-policy/federation-policy.go    |   3 -
 .../service-principal-federation-policy.go    |   3 -
 cmd/account/settings/settings.go              |   2 +
 cmd/workspace/alerts/alerts.go                |  15 +-
 cmd/workspace/catalogs/catalogs.go            |   1 +
 cmd/workspace/clean-rooms/clean-rooms.go      |   5 +-
 cmd/workspace/cmd.go                          |   6 +
 cmd/workspace/jobs/jobs.go                    |   1 +
 .../lakeview-embedded/lakeview-embedded.go    |  98 +++++
 cmd/workspace/queries/queries.go              |  15 +-
 .../query-execution/query-execution.go        | 245 ++++++++++++
 .../query-visualizations.go                   |  15 +-
 cmd/workspace/redash-config/redash-config.go  |  80 ++++
 .../serving-endpoints/serving-endpoints.go    |   3 +-
 go.mod                                        |   2 +-
 go.sum                                        |   4 +-
 24 files changed, 1109 insertions(+), 25 deletions(-)
 create mode 100755 cmd/account/budget-policy/budget-policy.go
 create mode 100755 cmd/account/enable-ip-access-lists/enable-ip-access-lists.go
 create mode 100755 cmd/workspace/lakeview-embedded/lakeview-embedded.go
 create mode 100755 cmd/workspace/query-execution/query-execution.go
 create mode 100755 cmd/workspace/redash-config/redash-config.go

diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha
index 588cf9d63..9a95107e8 100644
--- a/.codegen/_openapi_sha
+++ b/.codegen/_openapi_sha
@@ -1 +1 @@
-0be1b914249781b5e903b7676fd02255755bc851
\ No newline at end of file
+c72c58f97b950fcb924a90ef164bcb10cfcd5ece
\ No newline at end of file
diff --git a/.gitattributes b/.gitattributes
index ebe94ed8e..4b3715c93 100755
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,11 +1,13 @@
 cmd/account/access-control/access-control.go linguist-generated=true
 cmd/account/billable-usage/billable-usage.go linguist-generated=true
+cmd/account/budget-policy/budget-policy.go linguist-generated=true
 cmd/account/budgets/budgets.go linguist-generated=true
 cmd/account/cmd.go linguist-generated=true
 cmd/account/credentials/credentials.go linguist-generated=true
 cmd/account/csp-enablement-account/csp-enablement-account.go linguist-generated=true
 cmd/account/custom-app-integration/custom-app-integration.go linguist-generated=true
 cmd/account/disable-legacy-features/disable-legacy-features.go linguist-generated=true
+cmd/account/enable-ip-access-lists/enable-ip-access-lists.go linguist-generated=true
 cmd/account/encryption-keys/encryption-keys.go linguist-generated=true
 cmd/account/esm-enablement-account/esm-enablement-account.go linguist-generated=true
 cmd/account/federation-policy/federation-policy.go linguist-generated=true
@@ -75,6 +77,7 @@ cmd/workspace/instance-pools/instance-pools.go linguist-generated=true
 cmd/workspace/instance-profiles/instance-profiles.go linguist-generated=true
 cmd/workspace/ip-access-lists/ip-access-lists.go linguist-generated=true
 cmd/workspace/jobs/jobs.go linguist-generated=true
+cmd/workspace/lakeview-embedded/lakeview-embedded.go linguist-generated=true
 cmd/workspace/lakeview/lakeview.go linguist-generated=true
 cmd/workspace/libraries/libraries.go linguist-generated=true
 cmd/workspace/metastores/metastores.go linguist-generated=true
@@ -99,11 +102,13 @@ cmd/workspace/providers/providers.go linguist-generated=true
 cmd/workspace/quality-monitors/quality-monitors.go linguist-generated=true
 cmd/workspace/queries-legacy/queries-legacy.go linguist-generated=true
 cmd/workspace/queries/queries.go linguist-generated=true
+cmd/workspace/query-execution/query-execution.go linguist-generated=true
 cmd/workspace/query-history/query-history.go linguist-generated=true
 cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go linguist-generated=true
 cmd/workspace/query-visualizations/query-visualizations.go linguist-generated=true
 cmd/workspace/recipient-activation/recipient-activation.go linguist-generated=true
 cmd/workspace/recipients/recipients.go linguist-generated=true
+cmd/workspace/redash-config/redash-config.go linguist-generated=true
 cmd/workspace/registered-models/registered-models.go linguist-generated=true
 cmd/workspace/repos/repos.go linguist-generated=true
 cmd/workspace/resource-quotas/resource-quotas.go linguist-generated=true
diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml
index d9a0be50e..74cd06c66 100644
--- a/bundle/internal/schema/annotations_openapi.yml
+++ b/bundle/internal/schema/annotations_openapi.yml
@@ -281,6 +281,9 @@ github.com/databricks/cli/bundle/config/resources.Job:
   "parameters":
     "description": |-
       Job-level parameter definitions
+  "performance_target":
+    "description": |-
+      PerformanceTarget defines how performant or cost efficient the execution of run on serverless should be.
   "queue":
     "description": |-
       The queue settings of the job.
@@ -1818,6 +1821,17 @@ github.com/databricks/databricks-sdk-go/service/jobs.PauseStatus:
         UNPAUSED
       - |-
         PAUSED
+github.com/databricks/databricks-sdk-go/service/jobs.PerformanceTarget:
+  "_":
+    "description": |-
+      PerformanceTarget defines how performant (lower latency) or cost efficient the execution of run on serverless compute should be.
+      The performance mode on the job or pipeline should map to a performance setting that is passed to Cluster Manager
+      (see cluster-common PerformanceTarget).
+    "enum":
+      - |-
+        PERFORMANCE_OPTIMIZED
+      - |-
+        COST_OPTIMIZED
 github.com/databricks/databricks-sdk-go/service/jobs.PeriodicTriggerConfiguration:
   "interval":
     "description": |-
diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json
index 9d4304cd8..c3c31b58c 100644
--- a/bundle/schema/jsonschema.json
+++ b/bundle/schema/jsonschema.json
@@ -409,6 +409,10 @@
                         "description": "Job-level parameter definitions",
                         "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobParameterDefinition"
                       },
+                      "performance_target": {
+                        "description": "PerformanceTarget defines how performant or cost efficient the execution of run on serverless should be.",
+                        "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PerformanceTarget"
+                      },
                       "permissions": {
                         "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission"
                       },
@@ -3904,6 +3908,22 @@
                 }
               ]
             },
+            "jobs.PerformanceTarget": {
+              "oneOf": [
+                {
+                  "type": "string",
+                  "description": "PerformanceTarget defines how performant (lower latency) or cost efficient the execution of run on serverless compute should be.\nThe performance mode on the job or pipeline should map to a performance setting that is passed to Cluster Manager\n(see cluster-common PerformanceTarget).",
+                  "enum": [
+                    "PERFORMANCE_OPTIMIZED",
+                    "COST_OPTIMIZED"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
+              ]
+            },
             "jobs.PeriodicTriggerConfiguration": {
               "oneOf": [
                 {
diff --git a/cmd/account/budget-policy/budget-policy.go b/cmd/account/budget-policy/budget-policy.go
new file mode 100755
index 000000000..28b14ea91
--- /dev/null
+++ b/cmd/account/budget-policy/budget-policy.go
@@ -0,0 +1,373 @@
+// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT.
+
+package budget_policy
+
+import (
+	"fmt"
+
+	"github.com/databricks/cli/cmd/root"
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/flags"
+	"github.com/databricks/databricks-sdk-go/service/billing"
+	"github.com/spf13/cobra"
+)
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var cmdOverrides []func(*cobra.Command)
+
+func New() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "budget-policy",
+		Short:   `A service serves REST API about Budget policies.`,
+		Long:    `A service serves REST API about Budget policies`,
+		GroupID: "billing",
+		Annotations: map[string]string{
+			"package": "billing",
+		},
+	}
+
+	// Add methods
+	cmd.AddCommand(newCreate())
+	cmd.AddCommand(newDelete())
+	cmd.AddCommand(newGet())
+	cmd.AddCommand(newList())
+	cmd.AddCommand(newUpdate())
+
+	// Apply optional overrides to this command.
+	for _, fn := range cmdOverrides {
+		fn(cmd)
+	}
+
+	return cmd
+}
+
+// start create command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var createOverrides []func(
+	*cobra.Command,
+	*billing.CreateBudgetPolicyRequest,
+)
+
+func newCreate() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var createReq billing.CreateBudgetPolicyRequest
+	var createJson flags.JsonFlag
+
+	// TODO: short flags
+	cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`)
+
+	// TODO: array: custom_tags
+	cmd.Flags().StringVar(&createReq.PolicyName, "policy-name", createReq.PolicyName, `The name of the policy.`)
+	cmd.Flags().StringVar(&createReq.RequestId, "request-id", createReq.RequestId, `A unique identifier for this request.`)
+
+	cmd.Use = "create"
+	cmd.Short = `Create a budget policy.`
+	cmd.Long = `Create a budget policy.
+  
+  Creates a new policy.`
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(0)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustAccountClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		a := root.AccountClient(ctx)
+
+		if cmd.Flags().Changed("json") {
+			diags := createJson.Unmarshal(&createReq)
+			if diags.HasError() {
+				return diags.Error()
+			}
+			if len(diags) > 0 {
+				err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		response, err := a.BudgetPolicy.Create(ctx, createReq)
+		if err != nil {
+			return err
+		}
+		return cmdio.Render(ctx, response)
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range createOverrides {
+		fn(cmd, &createReq)
+	}
+
+	return cmd
+}
+
+// start delete command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var deleteOverrides []func(
+	*cobra.Command,
+	*billing.DeleteBudgetPolicyRequest,
+)
+
+func newDelete() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var deleteReq billing.DeleteBudgetPolicyRequest
+
+	// TODO: short flags
+
+	cmd.Use = "delete POLICY_ID"
+	cmd.Short = `Delete a budget policy.`
+	cmd.Long = `Delete a budget policy.
+  
+  Deletes a policy
+
+  Arguments:
+    POLICY_ID: The Id of the policy.`
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustAccountClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		a := root.AccountClient(ctx)
+
+		deleteReq.PolicyId = args[0]
+
+		err = a.BudgetPolicy.Delete(ctx, deleteReq)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range deleteOverrides {
+		fn(cmd, &deleteReq)
+	}
+
+	return cmd
+}
+
+// start get command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var getOverrides []func(
+	*cobra.Command,
+	*billing.GetBudgetPolicyRequest,
+)
+
+func newGet() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var getReq billing.GetBudgetPolicyRequest
+
+	// TODO: short flags
+
+	cmd.Use = "get POLICY_ID"
+	cmd.Short = `Get a budget policy.`
+	cmd.Long = `Get a budget policy.
+  
+  Retrieves a policy by it's ID.
+
+  Arguments:
+    POLICY_ID: The Id of the policy.`
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustAccountClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		a := root.AccountClient(ctx)
+
+		getReq.PolicyId = args[0]
+
+		response, err := a.BudgetPolicy.Get(ctx, getReq)
+		if err != nil {
+			return err
+		}
+		return cmdio.Render(ctx, response)
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range getOverrides {
+		fn(cmd, &getReq)
+	}
+
+	return cmd
+}
+
+// start list command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var listOverrides []func(
+	*cobra.Command,
+	*billing.ListBudgetPoliciesRequest,
+)
+
+func newList() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var listReq billing.ListBudgetPoliciesRequest
+
+	// TODO: short flags
+
+	// TODO: complex arg: filter_by
+	cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, `The maximum number of budget policies to return.`)
+	cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `A page token, received from a previous ListServerlessPolicies call.`)
+	// TODO: complex arg: sort_spec
+
+	cmd.Use = "list"
+	cmd.Short = `List policies.`
+	cmd.Long = `List policies.
+  
+  Lists all policies. Policies are returned in the alphabetically ascending
+  order of their names.`
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(0)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustAccountClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		a := root.AccountClient(ctx)
+
+		response := a.BudgetPolicy.List(ctx, listReq)
+		return cmdio.RenderIterator(ctx, response)
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range listOverrides {
+		fn(cmd, &listReq)
+	}
+
+	return cmd
+}
+
+// start update command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var updateOverrides []func(
+	*cobra.Command,
+	*billing.UpdateBudgetPolicyRequest,
+)
+
+func newUpdate() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var updateReq billing.UpdateBudgetPolicyRequest
+	updateReq.Policy = &billing.BudgetPolicy{}
+	var updateJson flags.JsonFlag
+
+	// TODO: short flags
+	cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`)
+
+	// TODO: array: custom_tags
+	cmd.Flags().StringVar(&updateReq.Policy.PolicyName, "policy-name", updateReq.Policy.PolicyName, `The name of the policy.`)
+
+	cmd.Use = "update POLICY_ID"
+	cmd.Short = `Update a budget policy.`
+	cmd.Long = `Update a budget policy.
+  
+  Updates a policy
+
+  Arguments:
+    POLICY_ID: The Id of the policy. This field is generated by Databricks and globally
+      unique.`
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		if cmd.Flags().Changed("json") {
+			err := root.ExactArgs(0)(cmd, args)
+			if err != nil {
+				return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'policy_id' in your JSON input")
+			}
+			return nil
+		}
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustAccountClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		a := root.AccountClient(ctx)
+
+		if cmd.Flags().Changed("json") {
+			diags := updateJson.Unmarshal(&updateReq.Policy)
+			if diags.HasError() {
+				return diags.Error()
+			}
+			if len(diags) > 0 {
+				err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags)
+				if err != nil {
+					return err
+				}
+			}
+		}
+		updateReq.PolicyId = args[0]
+
+		response, err := a.BudgetPolicy.Update(ctx, updateReq)
+		if err != nil {
+			return err
+		}
+		return cmdio.Render(ctx, response)
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range updateOverrides {
+		fn(cmd, &updateReq)
+	}
+
+	return cmd
+}
+
+// end service BudgetPolicy
diff --git a/cmd/account/cmd.go b/cmd/account/cmd.go
index f34966fd9..758e2af5e 100644
--- a/cmd/account/cmd.go
+++ b/cmd/account/cmd.go
@@ -7,6 +7,7 @@ import (
 
 	account_access_control "github.com/databricks/cli/cmd/account/access-control"
 	billable_usage "github.com/databricks/cli/cmd/account/billable-usage"
+	budget_policy "github.com/databricks/cli/cmd/account/budget-policy"
 	budgets "github.com/databricks/cli/cmd/account/budgets"
 	credentials "github.com/databricks/cli/cmd/account/credentials"
 	custom_app_integration "github.com/databricks/cli/cmd/account/custom-app-integration"
@@ -43,6 +44,7 @@ func New() *cobra.Command {
 
 	cmd.AddCommand(account_access_control.New())
 	cmd.AddCommand(billable_usage.New())
+	cmd.AddCommand(budget_policy.New())
 	cmd.AddCommand(credentials.New())
 	cmd.AddCommand(custom_app_integration.New())
 	cmd.AddCommand(encryption_keys.New())
diff --git a/cmd/account/custom-app-integration/custom-app-integration.go b/cmd/account/custom-app-integration/custom-app-integration.go
index 43e458bc6..61cfe0a09 100755
--- a/cmd/account/custom-app-integration/custom-app-integration.go
+++ b/cmd/account/custom-app-integration/custom-app-integration.go
@@ -65,6 +65,7 @@ func newCreate() *cobra.Command {
 	// TODO: array: redirect_urls
 	// TODO: array: scopes
 	// TODO: complex arg: token_access_policy
+	// TODO: array: user_authorized_scopes
 
 	cmd.Use = "create"
 	cmd.Short = `Create Custom OAuth App Integration.`
@@ -309,6 +310,7 @@ func newUpdate() *cobra.Command {
 	// TODO: array: redirect_urls
 	// TODO: array: scopes
 	// TODO: complex arg: token_access_policy
+	// TODO: array: user_authorized_scopes
 
 	cmd.Use = "update INTEGRATION_ID"
 	cmd.Short = `Updates Custom OAuth App Integration.`
diff --git a/cmd/account/enable-ip-access-lists/enable-ip-access-lists.go b/cmd/account/enable-ip-access-lists/enable-ip-access-lists.go
new file mode 100755
index 000000000..24d30c9c6
--- /dev/null
+++ b/cmd/account/enable-ip-access-lists/enable-ip-access-lists.go
@@ -0,0 +1,218 @@
+// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT.
+
+package enable_ip_access_lists
+
+import (
+	"fmt"
+
+	"github.com/databricks/cli/cmd/root"
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/flags"
+	"github.com/databricks/databricks-sdk-go/service/settings"
+	"github.com/spf13/cobra"
+)
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var cmdOverrides []func(*cobra.Command)
+
+func New() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "enable-ip-access-lists",
+		Short: `Controls the enforcement of IP access lists for accessing the account console.`,
+		Long: `Controls the enforcement of IP access lists for accessing the account console.
+  Allowing you to enable or disable restricted access based on IP addresses.`,
+
+		// This service is being previewed; hide from help output.
+		Hidden: true,
+	}
+
+	// Add methods
+	cmd.AddCommand(newDelete())
+	cmd.AddCommand(newGet())
+	cmd.AddCommand(newUpdate())
+
+	// Apply optional overrides to this command.
+	for _, fn := range cmdOverrides {
+		fn(cmd)
+	}
+
+	return cmd
+}
+
+// start delete command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var deleteOverrides []func(
+	*cobra.Command,
+	*settings.DeleteAccountIpAccessEnableRequest,
+)
+
+func newDelete() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var deleteReq settings.DeleteAccountIpAccessEnableRequest
+
+	// TODO: short flags
+
+	cmd.Flags().StringVar(&deleteReq.Etag, "etag", deleteReq.Etag, `etag used for versioning.`)
+
+	cmd.Use = "delete"
+	cmd.Short = `Delete the account IP access toggle setting.`
+	cmd.Long = `Delete the account IP access toggle setting.
+  
+  Reverts the value of the account IP access toggle setting to default (ON)`
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(0)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustAccountClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		a := root.AccountClient(ctx)
+
+		response, err := a.Settings.EnableIpAccessLists().Delete(ctx, deleteReq)
+		if err != nil {
+			return err
+		}
+		return cmdio.Render(ctx, response)
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range deleteOverrides {
+		fn(cmd, &deleteReq)
+	}
+
+	return cmd
+}
+
+// start get command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var getOverrides []func(
+	*cobra.Command,
+	*settings.GetAccountIpAccessEnableRequest,
+)
+
+func newGet() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var getReq settings.GetAccountIpAccessEnableRequest
+
+	// TODO: short flags
+
+	cmd.Flags().StringVar(&getReq.Etag, "etag", getReq.Etag, `etag used for versioning.`)
+
+	cmd.Use = "get"
+	cmd.Short = `Get the account IP access toggle setting.`
+	cmd.Long = `Get the account IP access toggle setting.
+  
+  Gets the value of the account IP access toggle setting.`
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(0)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustAccountClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		a := root.AccountClient(ctx)
+
+		response, err := a.Settings.EnableIpAccessLists().Get(ctx, getReq)
+		if err != nil {
+			return err
+		}
+		return cmdio.Render(ctx, response)
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range getOverrides {
+		fn(cmd, &getReq)
+	}
+
+	return cmd
+}
+
+// start update command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var updateOverrides []func(
+	*cobra.Command,
+	*settings.UpdateAccountIpAccessEnableRequest,
+)
+
+func newUpdate() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var updateReq settings.UpdateAccountIpAccessEnableRequest
+	var updateJson flags.JsonFlag
+
+	// TODO: short flags
+	cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`)
+
+	cmd.Use = "update"
+	cmd.Short = `Update the account IP access toggle setting.`
+	cmd.Long = `Update the account IP access toggle setting.
+  
+  Updates the value of the account IP access toggle setting.`
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.PreRunE = root.MustAccountClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		a := root.AccountClient(ctx)
+
+		if cmd.Flags().Changed("json") {
+			diags := updateJson.Unmarshal(&updateReq)
+			if diags.HasError() {
+				return diags.Error()
+			}
+			if len(diags) > 0 {
+				err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags)
+				if err != nil {
+					return err
+				}
+			}
+		} else {
+			return fmt.Errorf("please provide command input in JSON format by specifying the --json flag")
+		}
+
+		response, err := a.Settings.EnableIpAccessLists().Update(ctx, updateReq)
+		if err != nil {
+			return err
+		}
+		return cmdio.Render(ctx, response)
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range updateOverrides {
+		fn(cmd, &updateReq)
+	}
+
+	return cmd
+}
+
+// end service EnableIpAccessLists
diff --git a/cmd/account/federation-policy/federation-policy.go b/cmd/account/federation-policy/federation-policy.go
index e47bf8324..ad45c0405 100755
--- a/cmd/account/federation-policy/federation-policy.go
+++ b/cmd/account/federation-policy/federation-policy.go
@@ -71,9 +71,6 @@ func New() *cobra.Command {
 		Annotations: map[string]string{
 			"package": "oauth2",
 		},
-
-		// This service is being previewed; hide from help output.
-		Hidden: true,
 	}
 
 	// Add methods
diff --git a/cmd/account/service-principal-federation-policy/service-principal-federation-policy.go b/cmd/account/service-principal-federation-policy/service-principal-federation-policy.go
index df36de239..451523b7e 100755
--- a/cmd/account/service-principal-federation-policy/service-principal-federation-policy.go
+++ b/cmd/account/service-principal-federation-policy/service-principal-federation-policy.go
@@ -78,9 +78,6 @@ func New() *cobra.Command {
 		Annotations: map[string]string{
 			"package": "oauth2",
 		},
-
-		// This service is being previewed; hide from help output.
-		Hidden: true,
 	}
 
 	// Add methods
diff --git a/cmd/account/settings/settings.go b/cmd/account/settings/settings.go
index 9a9cd44bf..cd30743f7 100755
--- a/cmd/account/settings/settings.go
+++ b/cmd/account/settings/settings.go
@@ -7,6 +7,7 @@ import (
 
 	csp_enablement_account "github.com/databricks/cli/cmd/account/csp-enablement-account"
 	disable_legacy_features "github.com/databricks/cli/cmd/account/disable-legacy-features"
+	enable_ip_access_lists "github.com/databricks/cli/cmd/account/enable-ip-access-lists"
 	esm_enablement_account "github.com/databricks/cli/cmd/account/esm-enablement-account"
 	personal_compute "github.com/databricks/cli/cmd/account/personal-compute"
 )
@@ -29,6 +30,7 @@ func New() *cobra.Command {
 	// Add subservices
 	cmd.AddCommand(csp_enablement_account.New())
 	cmd.AddCommand(disable_legacy_features.New())
+	cmd.AddCommand(enable_ip_access_lists.New())
 	cmd.AddCommand(esm_enablement_account.New())
 	cmd.AddCommand(personal_compute.New())
 
diff --git a/cmd/workspace/alerts/alerts.go b/cmd/workspace/alerts/alerts.go
index fcf18652b..79467c405 100755
--- a/cmd/workspace/alerts/alerts.go
+++ b/cmd/workspace/alerts/alerts.go
@@ -335,10 +335,17 @@ func newUpdate() *cobra.Command {
 
   Arguments:
     ID: 
-    UPDATE_MASK: Field mask is required to be passed into the PATCH request. Field mask
-      specifies which fields of the setting payload will be updated. The field
-      mask needs to be supplied as single string. To specify multiple fields in
-      the field mask, use comma as the separator (no space).`
+    UPDATE_MASK: The field mask must be a single string, with multiple fields separated by
+      commas (no spaces). The field path is relative to the resource object,
+      using a dot (.) to navigate sub-fields (e.g., author.given_name).
+      Specification of elements in sequence or map fields is not allowed, as
+      only the entire collection field can be specified. Field names must
+      exactly match the resource field names.
+      
+      A field mask of * indicates full replacement. It’s recommended to
+      always explicitly list the fields being updated and avoid using *
+      wildcards, as it can lead to unintended results if the API changes in the
+      future.`
 
 	cmd.Annotations = make(map[string]string)
 
diff --git a/cmd/workspace/catalogs/catalogs.go b/cmd/workspace/catalogs/catalogs.go
index 9294c192b..ce37b6d54 100755
--- a/cmd/workspace/catalogs/catalogs.go
+++ b/cmd/workspace/catalogs/catalogs.go
@@ -342,6 +342,7 @@ func newUpdate() *cobra.Command {
 	cmd.Flags().Var(&updateReq.EnablePredictiveOptimization, "enable-predictive-optimization", `Whether predictive optimization should be enabled for this object and objects under it. Supported values: [DISABLE, ENABLE, INHERIT]`)
 	cmd.Flags().Var(&updateReq.IsolationMode, "isolation-mode", `Whether the current securable is accessible from all workspaces or a specific set of workspaces. Supported values: [ISOLATED, OPEN]`)
 	cmd.Flags().StringVar(&updateReq.NewName, "new-name", updateReq.NewName, `New name for the catalog.`)
+	// TODO: map via StringToStringVar: options
 	cmd.Flags().StringVar(&updateReq.Owner, "owner", updateReq.Owner, `Username of current owner of catalog.`)
 	// TODO: map via StringToStringVar: properties
 
diff --git a/cmd/workspace/clean-rooms/clean-rooms.go b/cmd/workspace/clean-rooms/clean-rooms.go
index 053e41e8a..4fe61d56b 100755
--- a/cmd/workspace/clean-rooms/clean-rooms.go
+++ b/cmd/workspace/clean-rooms/clean-rooms.go
@@ -75,8 +75,9 @@ func newCreate() *cobra.Command {
   Create a new clean room with the specified collaborators. This method is
   asynchronous; the returned name field inside the clean_room field can be used
   to poll the clean room status, using the :method:cleanrooms/get method. When
-  this method returns, the cluster will be in a PROVISIONING state. The cluster
-  will be usable once it enters an ACTIVE state.
+  this method returns, the clean room will be in a PROVISIONING state, with only
+  name, owner, comment, created_at and status populated. The clean room will be
+  usable once it enters an ACTIVE state.
   
   The caller must be a metastore admin or have the **CREATE_CLEAN_ROOM**
   privilege on the metastore.`
diff --git a/cmd/workspace/cmd.go b/cmd/workspace/cmd.go
index c447bd736..2bd3c59a5 100755
--- a/cmd/workspace/cmd.go
+++ b/cmd/workspace/cmd.go
@@ -39,6 +39,7 @@ import (
 	ip_access_lists "github.com/databricks/cli/cmd/workspace/ip-access-lists"
 	jobs "github.com/databricks/cli/cmd/workspace/jobs"
 	lakeview "github.com/databricks/cli/cmd/workspace/lakeview"
+	lakeview_embedded "github.com/databricks/cli/cmd/workspace/lakeview-embedded"
 	libraries "github.com/databricks/cli/cmd/workspace/libraries"
 	metastores "github.com/databricks/cli/cmd/workspace/metastores"
 	model_registry "github.com/databricks/cli/cmd/workspace/model-registry"
@@ -62,11 +63,13 @@ import (
 	quality_monitors "github.com/databricks/cli/cmd/workspace/quality-monitors"
 	queries "github.com/databricks/cli/cmd/workspace/queries"
 	queries_legacy "github.com/databricks/cli/cmd/workspace/queries-legacy"
+	query_execution "github.com/databricks/cli/cmd/workspace/query-execution"
 	query_history "github.com/databricks/cli/cmd/workspace/query-history"
 	query_visualizations "github.com/databricks/cli/cmd/workspace/query-visualizations"
 	query_visualizations_legacy "github.com/databricks/cli/cmd/workspace/query-visualizations-legacy"
 	recipient_activation "github.com/databricks/cli/cmd/workspace/recipient-activation"
 	recipients "github.com/databricks/cli/cmd/workspace/recipients"
+	redash_config "github.com/databricks/cli/cmd/workspace/redash-config"
 	registered_models "github.com/databricks/cli/cmd/workspace/registered-models"
 	repos "github.com/databricks/cli/cmd/workspace/repos"
 	resource_quotas "github.com/databricks/cli/cmd/workspace/resource-quotas"
@@ -133,6 +136,7 @@ func All() []*cobra.Command {
 	out = append(out, ip_access_lists.New())
 	out = append(out, jobs.New())
 	out = append(out, lakeview.New())
+	out = append(out, lakeview_embedded.New())
 	out = append(out, libraries.New())
 	out = append(out, metastores.New())
 	out = append(out, model_registry.New())
@@ -156,11 +160,13 @@ func All() []*cobra.Command {
 	out = append(out, quality_monitors.New())
 	out = append(out, queries.New())
 	out = append(out, queries_legacy.New())
+	out = append(out, query_execution.New())
 	out = append(out, query_history.New())
 	out = append(out, query_visualizations.New())
 	out = append(out, query_visualizations_legacy.New())
 	out = append(out, recipient_activation.New())
 	out = append(out, recipients.New())
+	out = append(out, redash_config.New())
 	out = append(out, registered_models.New())
 	out = append(out, repos.New())
 	out = append(out, resource_quotas.New())
diff --git a/cmd/workspace/jobs/jobs.go b/cmd/workspace/jobs/jobs.go
index 38a88f014..0f911d400 100755
--- a/cmd/workspace/jobs/jobs.go
+++ b/cmd/workspace/jobs/jobs.go
@@ -1354,6 +1354,7 @@ func newRunNow() *cobra.Command {
 	// TODO: map via StringToStringVar: job_parameters
 	// TODO: map via StringToStringVar: notebook_params
 	// TODO: array: only
+	cmd.Flags().Var(&runNowReq.PerformanceTarget, "performance-target", `PerformanceTarget defines how performant or cost efficient the execution of run on serverless compute should be. Supported values: [COST_OPTIMIZED, PERFORMANCE_OPTIMIZED]`)
 	// TODO: complex arg: pipeline_params
 	// TODO: map via StringToStringVar: python_named_params
 	// TODO: array: python_params
diff --git a/cmd/workspace/lakeview-embedded/lakeview-embedded.go b/cmd/workspace/lakeview-embedded/lakeview-embedded.go
new file mode 100755
index 000000000..ef04c2c13
--- /dev/null
+++ b/cmd/workspace/lakeview-embedded/lakeview-embedded.go
@@ -0,0 +1,98 @@
+// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT.
+
+package lakeview_embedded
+
+import (
+	"github.com/databricks/cli/cmd/root"
+	"github.com/databricks/databricks-sdk-go/service/dashboards"
+	"github.com/spf13/cobra"
+)
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var cmdOverrides []func(*cobra.Command)
+
+func New() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "lakeview-embedded",
+		Short:   `Token-based Lakeview APIs for embedding dashboards in external applications.`,
+		Long:    `Token-based Lakeview APIs for embedding dashboards in external applications.`,
+		GroupID: "dashboards",
+		Annotations: map[string]string{
+			"package": "dashboards",
+		},
+	}
+
+	// Add methods
+	cmd.AddCommand(newGetPublishedDashboardEmbedded())
+
+	// Apply optional overrides to this command.
+	for _, fn := range cmdOverrides {
+		fn(cmd)
+	}
+
+	return cmd
+}
+
+// start get-published-dashboard-embedded command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var getPublishedDashboardEmbeddedOverrides []func(
+	*cobra.Command,
+	*dashboards.GetPublishedDashboardEmbeddedRequest,
+)
+
+func newGetPublishedDashboardEmbedded() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var getPublishedDashboardEmbeddedReq dashboards.GetPublishedDashboardEmbeddedRequest
+
+	// TODO: short flags
+
+	cmd.Use = "get-published-dashboard-embedded DASHBOARD_ID"
+	cmd.Short = `Read a published dashboard in an embedded ui.`
+	cmd.Long = `Read a published dashboard in an embedded ui.
+  
+  Get the current published dashboard within an embedded context.
+
+  Arguments:
+    DASHBOARD_ID: UUID identifying the published dashboard.`
+
+	// This command is being previewed; hide from help output.
+	cmd.Hidden = true
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustWorkspaceClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		w := root.WorkspaceClient(ctx)
+
+		getPublishedDashboardEmbeddedReq.DashboardId = args[0]
+
+		err = w.LakeviewEmbedded.GetPublishedDashboardEmbedded(ctx, getPublishedDashboardEmbeddedReq)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range getPublishedDashboardEmbeddedOverrides {
+		fn(cmd, &getPublishedDashboardEmbeddedReq)
+	}
+
+	return cmd
+}
+
+// end service LakeviewEmbedded
diff --git a/cmd/workspace/queries/queries.go b/cmd/workspace/queries/queries.go
index 208f887da..bf74bb3f5 100755
--- a/cmd/workspace/queries/queries.go
+++ b/cmd/workspace/queries/queries.go
@@ -406,10 +406,17 @@ func newUpdate() *cobra.Command {
 
   Arguments:
     ID: 
-    UPDATE_MASK: Field mask is required to be passed into the PATCH request. Field mask
-      specifies which fields of the setting payload will be updated. The field
-      mask needs to be supplied as single string. To specify multiple fields in
-      the field mask, use comma as the separator (no space).`
+    UPDATE_MASK: The field mask must be a single string, with multiple fields separated by
+      commas (no spaces). The field path is relative to the resource object,
+      using a dot (.) to navigate sub-fields (e.g., author.given_name).
+      Specification of elements in sequence or map fields is not allowed, as
+      only the entire collection field can be specified. Field names must
+      exactly match the resource field names.
+      
+      A field mask of * indicates full replacement. It’s recommended to
+      always explicitly list the fields being updated and avoid using *
+      wildcards, as it can lead to unintended results if the API changes in the
+      future.`
 
 	cmd.Annotations = make(map[string]string)
 
diff --git a/cmd/workspace/query-execution/query-execution.go b/cmd/workspace/query-execution/query-execution.go
new file mode 100755
index 000000000..ebbb90f89
--- /dev/null
+++ b/cmd/workspace/query-execution/query-execution.go
@@ -0,0 +1,245 @@
+// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT.
+
+package query_execution
+
+import (
+	"fmt"
+
+	"github.com/databricks/cli/cmd/root"
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/flags"
+	"github.com/databricks/databricks-sdk-go/service/dashboards"
+	"github.com/spf13/cobra"
+)
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var cmdOverrides []func(*cobra.Command)
+
+func New() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "query-execution",
+		Short:   `Query execution APIs for AI / BI Dashboards.`,
+		Long:    `Query execution APIs for AI / BI Dashboards`,
+		GroupID: "dashboards",
+		Annotations: map[string]string{
+			"package": "dashboards",
+		},
+
+		// This service is being previewed; hide from help output.
+		Hidden: true,
+	}
+
+	// Add methods
+	cmd.AddCommand(newCancelPublishedQueryExecution())
+	cmd.AddCommand(newExecutePublishedDashboardQuery())
+	cmd.AddCommand(newPollPublishedQueryStatus())
+
+	// Apply optional overrides to this command.
+	for _, fn := range cmdOverrides {
+		fn(cmd)
+	}
+
+	return cmd
+}
+
+// start cancel-published-query-execution command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var cancelPublishedQueryExecutionOverrides []func(
+	*cobra.Command,
+	*dashboards.CancelPublishedQueryExecutionRequest,
+)
+
+func newCancelPublishedQueryExecution() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var cancelPublishedQueryExecutionReq dashboards.CancelPublishedQueryExecutionRequest
+
+	// TODO: short flags
+
+	// TODO: array: tokens
+
+	cmd.Use = "cancel-published-query-execution DASHBOARD_NAME DASHBOARD_REVISION_ID"
+	cmd.Short = `Cancel the results for the a query for a published, embedded dashboard.`
+	cmd.Long = `Cancel the results for the a query for a published, embedded dashboard.`
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(2)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustWorkspaceClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		w := root.WorkspaceClient(ctx)
+
+		cancelPublishedQueryExecutionReq.DashboardName = args[0]
+		cancelPublishedQueryExecutionReq.DashboardRevisionId = args[1]
+
+		response, err := w.QueryExecution.CancelPublishedQueryExecution(ctx, cancelPublishedQueryExecutionReq)
+		if err != nil {
+			return err
+		}
+		return cmdio.Render(ctx, response)
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range cancelPublishedQueryExecutionOverrides {
+		fn(cmd, &cancelPublishedQueryExecutionReq)
+	}
+
+	return cmd
+}
+
+// start execute-published-dashboard-query command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var executePublishedDashboardQueryOverrides []func(
+	*cobra.Command,
+	*dashboards.ExecutePublishedDashboardQueryRequest,
+)
+
+func newExecutePublishedDashboardQuery() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var executePublishedDashboardQueryReq dashboards.ExecutePublishedDashboardQueryRequest
+	var executePublishedDashboardQueryJson flags.JsonFlag
+
+	// TODO: short flags
+	cmd.Flags().Var(&executePublishedDashboardQueryJson, "json", `either inline JSON string or @path/to/file.json with request body`)
+
+	cmd.Flags().StringVar(&executePublishedDashboardQueryReq.OverrideWarehouseId, "override-warehouse-id", executePublishedDashboardQueryReq.OverrideWarehouseId, `A dashboard schedule can override the warehouse used as compute for processing the published dashboard queries.`)
+
+	cmd.Use = "execute-published-dashboard-query DASHBOARD_NAME DASHBOARD_REVISION_ID"
+	cmd.Short = `Execute a query for a published dashboard.`
+	cmd.Long = `Execute a query for a published dashboard.
+
+  Arguments:
+    DASHBOARD_NAME: Dashboard name and revision_id is required to retrieve
+      PublishedDatasetDataModel which contains the list of datasets,
+      warehouse_id, and embedded_credentials
+    DASHBOARD_REVISION_ID: `
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		if cmd.Flags().Changed("json") {
+			err := root.ExactArgs(0)(cmd, args)
+			if err != nil {
+				return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'dashboard_name', 'dashboard_revision_id' in your JSON input")
+			}
+			return nil
+		}
+		check := root.ExactArgs(2)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustWorkspaceClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		w := root.WorkspaceClient(ctx)
+
+		if cmd.Flags().Changed("json") {
+			diags := executePublishedDashboardQueryJson.Unmarshal(&executePublishedDashboardQueryReq)
+			if diags.HasError() {
+				return diags.Error()
+			}
+			if len(diags) > 0 {
+				err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags)
+				if err != nil {
+					return err
+				}
+			}
+		}
+		if !cmd.Flags().Changed("json") {
+			executePublishedDashboardQueryReq.DashboardName = args[0]
+		}
+		if !cmd.Flags().Changed("json") {
+			executePublishedDashboardQueryReq.DashboardRevisionId = args[1]
+		}
+
+		err = w.QueryExecution.ExecutePublishedDashboardQuery(ctx, executePublishedDashboardQueryReq)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range executePublishedDashboardQueryOverrides {
+		fn(cmd, &executePublishedDashboardQueryReq)
+	}
+
+	return cmd
+}
+
+// start poll-published-query-status command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var pollPublishedQueryStatusOverrides []func(
+	*cobra.Command,
+	*dashboards.PollPublishedQueryStatusRequest,
+)
+
+func newPollPublishedQueryStatus() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var pollPublishedQueryStatusReq dashboards.PollPublishedQueryStatusRequest
+
+	// TODO: short flags
+
+	// TODO: array: tokens
+
+	cmd.Use = "poll-published-query-status DASHBOARD_NAME DASHBOARD_REVISION_ID"
+	cmd.Short = `Poll the results for the a query for a published, embedded dashboard.`
+	cmd.Long = `Poll the results for the a query for a published, embedded dashboard.`
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(2)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustWorkspaceClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		w := root.WorkspaceClient(ctx)
+
+		pollPublishedQueryStatusReq.DashboardName = args[0]
+		pollPublishedQueryStatusReq.DashboardRevisionId = args[1]
+
+		response, err := w.QueryExecution.PollPublishedQueryStatus(ctx, pollPublishedQueryStatusReq)
+		if err != nil {
+			return err
+		}
+		return cmdio.Render(ctx, response)
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range pollPublishedQueryStatusOverrides {
+		fn(cmd, &pollPublishedQueryStatusReq)
+	}
+
+	return cmd
+}
+
+// end service QueryExecution
diff --git a/cmd/workspace/query-visualizations/query-visualizations.go b/cmd/workspace/query-visualizations/query-visualizations.go
index 621661952..2d50229ba 100755
--- a/cmd/workspace/query-visualizations/query-visualizations.go
+++ b/cmd/workspace/query-visualizations/query-visualizations.go
@@ -198,10 +198,17 @@ func newUpdate() *cobra.Command {
 
   Arguments:
     ID: 
-    UPDATE_MASK: Field mask is required to be passed into the PATCH request. Field mask
-      specifies which fields of the setting payload will be updated. The field
-      mask needs to be supplied as single string. To specify multiple fields in
-      the field mask, use comma as the separator (no space).`
+    UPDATE_MASK: The field mask must be a single string, with multiple fields separated by
+      commas (no spaces). The field path is relative to the resource object,
+      using a dot (.) to navigate sub-fields (e.g., author.given_name).
+      Specification of elements in sequence or map fields is not allowed, as
+      only the entire collection field can be specified. Field names must
+      exactly match the resource field names.
+      
+      A field mask of * indicates full replacement. It’s recommended to
+      always explicitly list the fields being updated and avoid using *
+      wildcards, as it can lead to unintended results if the API changes in the
+      future.`
 
 	cmd.Annotations = make(map[string]string)
 
diff --git a/cmd/workspace/redash-config/redash-config.go b/cmd/workspace/redash-config/redash-config.go
new file mode 100755
index 000000000..1a0f37759
--- /dev/null
+++ b/cmd/workspace/redash-config/redash-config.go
@@ -0,0 +1,80 @@
+// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT.
+
+package redash_config
+
+import (
+	"github.com/databricks/cli/cmd/root"
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/spf13/cobra"
+)
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var cmdOverrides []func(*cobra.Command)
+
+func New() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "redash-config",
+		Short:   `Redash V2 service for workspace configurations (internal).`,
+		Long:    `Redash V2 service for workspace configurations (internal)`,
+		GroupID: "sql",
+		Annotations: map[string]string{
+			"package": "sql",
+		},
+
+		// This service is being previewed; hide from help output.
+		Hidden: true,
+	}
+
+	// Add methods
+	cmd.AddCommand(newGetConfig())
+
+	// Apply optional overrides to this command.
+	for _, fn := range cmdOverrides {
+		fn(cmd)
+	}
+
+	return cmd
+}
+
+// start get-config command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var getConfigOverrides []func(
+	*cobra.Command,
+)
+
+func newGetConfig() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	cmd.Use = "get-config"
+	cmd.Short = `Read workspace configuration for Redash-v2.`
+	cmd.Long = `Read workspace configuration for Redash-v2.`
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.PreRunE = root.MustWorkspaceClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		w := root.WorkspaceClient(ctx)
+		response, err := w.RedashConfig.GetConfig(ctx)
+		if err != nil {
+			return err
+		}
+		return cmdio.Render(ctx, response)
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range getConfigOverrides {
+		fn(cmd)
+	}
+
+	return cmd
+}
+
+// end service RedashConfig
diff --git a/cmd/workspace/serving-endpoints/serving-endpoints.go b/cmd/workspace/serving-endpoints/serving-endpoints.go
index 034133623..645111646 100755
--- a/cmd/workspace/serving-endpoints/serving-endpoints.go
+++ b/cmd/workspace/serving-endpoints/serving-endpoints.go
@@ -642,7 +642,8 @@ func newHttpRequest() *cobra.Command {
 		if err != nil {
 			return err
 		}
-		return cmdio.Render(ctx, response)
+		defer response.Contents.Close()
+		return cmdio.Render(ctx, response.Contents)
 	}
 
 	// Disable completions since they are not applicable.
diff --git a/go.mod b/go.mod
index 662fcd40b..db3261beb 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/BurntSushi/toml v1.4.0 // MIT
 	github.com/Masterminds/semver/v3 v3.3.1 // MIT
 	github.com/briandowns/spinner v1.23.1 // Apache 2.0
-	github.com/databricks/databricks-sdk-go v0.56.1 // Apache 2.0
+	github.com/databricks/databricks-sdk-go v0.57.0 // Apache 2.0
 	github.com/fatih/color v1.18.0 // MIT
 	github.com/google/uuid v1.6.0 // BSD-3-Clause
 	github.com/hashicorp/go-version v1.7.0 // MPL 2.0
diff --git a/go.sum b/go.sum
index bffc3b53d..d6102a0d1 100644
--- a/go.sum
+++ b/go.sum
@@ -34,8 +34,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cyphar/filepath-securejoin v0.2.5 h1:6iR5tXJ/e6tJZzzdMc1km3Sa7RRIVBKAK32O2s7AYfo=
 github.com/cyphar/filepath-securejoin v0.2.5/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
-github.com/databricks/databricks-sdk-go v0.56.1 h1:sgweTRvAQaI8EPrfDnVdAB0lNX6L5uTT720SlMMQI2U=
-github.com/databricks/databricks-sdk-go v0.56.1/go.mod h1:JpLizplEs+up9/Z4Xf2x++o3sM9eTTWFGzIXAptKJzI=
+github.com/databricks/databricks-sdk-go v0.57.0 h1:Vs3a+Zmg403er4+xpD7ZTQWm7e51d2q3yYEyIIgvtYw=
+github.com/databricks/databricks-sdk-go v0.57.0/go.mod h1:JpLizplEs+up9/Z4Xf2x++o3sM9eTTWFGzIXAptKJzI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

From d282f33a22072525db102ba5b2f5952f001f70ed Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Mon, 10 Feb 2025 14:00:49 +0100
Subject: [PATCH 222/247] Append newline to "-o json" for validate/summary/run
 (#2326)

## Changes
- Insert newline after rendering indented JSON in bundle
validate/summary/run.
- This prevents "No newline at end of file" message in various cases,
for example when switching between recording raw output of the command
to output processed by jq, since jq does add a newline or when running
diff in acceptance tests.

## Tests
Manually running validate:

```
~/work/dabs_cuj_brickfood % ../cli/cli-main bundle validate -o json | tail -n 2  # without change
Error: root_path must start with '~/' or contain the current username to ensure uniqueness when using 'mode: development'

  }
}%
~/work/dabs_cuj_brickfood % ../cli/cli bundle validate -o json | tail -n 2  # with change
Error: root_path must start with '~/' or contain the current username to ensure uniqueness when using 'mode: development'

  }
}
~/work/dabs_cuj_brickfood %
```

Via #2316 -- see cleaner output there.
---
 acceptance/bundle/variables/host/output.txt | 1 +
 cmd/bundle/run.go                           | 1 +
 cmd/bundle/summary.go                       | 1 +
 cmd/bundle/validate.go                      | 4 +++-
 4 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/acceptance/bundle/variables/host/output.txt b/acceptance/bundle/variables/host/output.txt
index 63c41426a..df0a4527a 100644
--- a/acceptance/bundle/variables/host/output.txt
+++ b/acceptance/bundle/variables/host/output.txt
@@ -23,6 +23,7 @@ Error: failed during request visitor: parse "https://${var.host}": invalid chara
     "host": "${var.host}"
   }
 }
+
 Exit code: 1
 
 >>> errcode [CLI] bundle validate
diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go
index df35d7222..ffb9c1b88 100644
--- a/cmd/bundle/run.go
+++ b/cmd/bundle/run.go
@@ -173,6 +173,7 @@ task or a Python wheel task, the second example applies.
 				if err != nil {
 					return err
 				}
+				_, _ = cmd.OutOrStdout().Write([]byte{'\n'})
 			default:
 				return fmt.Errorf("unknown output type %s", root.OutputType(cmd))
 			}
diff --git a/cmd/bundle/summary.go b/cmd/bundle/summary.go
index 7c669c845..2871c82ff 100644
--- a/cmd/bundle/summary.go
+++ b/cmd/bundle/summary.go
@@ -74,6 +74,7 @@ func newSummaryCommand() *cobra.Command {
 				return err
 			}
 			_, _ = cmd.OutOrStdout().Write(buf)
+			_, _ = cmd.OutOrStdout().Write([]byte{'\n'})
 		default:
 			return fmt.Errorf("unknown output type %s", root.OutputType(cmd))
 		}
diff --git a/cmd/bundle/validate.go b/cmd/bundle/validate.go
index 41fa87f30..c45453af6 100644
--- a/cmd/bundle/validate.go
+++ b/cmd/bundle/validate.go
@@ -20,7 +20,9 @@ func renderJsonOutput(cmd *cobra.Command, b *bundle.Bundle) error {
 	if err != nil {
 		return err
 	}
-	_, _ = cmd.OutOrStdout().Write(buf)
+	out := cmd.OutOrStdout()
+	_, _ = out.Write(buf)
+	_, _ = out.Write([]byte{'\n'})
 	return nil
 }
 

From ddedc4272d0f07eb49b2d23bea0dfba5f0773d33 Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Mon, 10 Feb 2025 18:35:12 +0530
Subject: [PATCH 223/247] Return 501 status code when API stub is not
 implemented (#2327)

## Changes
Addresses feedback from
https://github.com/databricks/cli/pull/2292#discussion_r1946846865

## Tests
Manually, confirmed that unstubbed API calls still cause acceptance
tests to fail.
---
 libs/testserver/server.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libs/testserver/server.go b/libs/testserver/server.go
index d0c340c12..d5877e90e 100644
--- a/libs/testserver/server.go
+++ b/libs/testserver/server.go
@@ -76,7 +76,7 @@ Response.StatusCode = <response status-code here>
 
 		return apierr.APIError{
 			Message: "No stub found for pattern: " + pattern,
-		}, http.StatusNotFound
+		}, http.StatusNotImplemented
 	})
 
 	return s

From 6953a84db6d85fc72e7b5bb6347acb59d68bbfa5 Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Mon, 10 Feb 2025 19:03:27 +0530
Subject: [PATCH 224/247] Serialize recorded requests with indentation in
 acceptance tests (#2329)

## Changes
This PR indents the recorded requests to make them easier to review.
They can still be parsed using jq.

## Tests
Existing tests.
---
 acceptance/acceptance_test.go                    |  2 +-
 .../workspace/jobs/create-error/out.requests.txt |  8 +++++++-
 .../workspace/jobs/create/out.requests.txt       | 16 +++++++++++++++-
 3 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 241ab42be..b05b10f47 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -318,7 +318,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		require.NoError(t, err)
 
 		for _, req := range server.Requests {
-			reqJson, err := json.Marshal(req)
+			reqJson, err := json.MarshalIndent(req, "", "  ")
 			require.NoError(t, err)
 
 			reqJsonWithRepls := repls.Replace(string(reqJson))
diff --git a/acceptance/workspace/jobs/create-error/out.requests.txt b/acceptance/workspace/jobs/create-error/out.requests.txt
index b22876b70..30f104fd1 100644
--- a/acceptance/workspace/jobs/create-error/out.requests.txt
+++ b/acceptance/workspace/jobs/create-error/out.requests.txt
@@ -1 +1,7 @@
-{"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}}
+{
+  "method": "POST",
+  "path": "/api/2.1/jobs/create",
+  "body": {
+    "name": "abc"
+  }
+}
diff --git a/acceptance/workspace/jobs/create/out.requests.txt b/acceptance/workspace/jobs/create/out.requests.txt
index 2510762db..1d200a547 100644
--- a/acceptance/workspace/jobs/create/out.requests.txt
+++ b/acceptance/workspace/jobs/create/out.requests.txt
@@ -1 +1,15 @@
-{"headers":{"Authorization":["Bearer [DATABRICKS_TOKEN]"],"User-Agent":["cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"]},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}}
+{
+  "headers": {
+    "Authorization": [
+      "Bearer [DATABRICKS_TOKEN]"
+    ],
+    "User-Agent": [
+      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"
+    ]
+  },
+  "method": "POST",
+  "path": "/api/2.1/jobs/create",
+  "body": {
+    "name": "abc"
+  }
+}

From 4bc231ad4f0d0bead1c1c26e11dedd1480d4e892 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 10 Feb 2025 14:58:18 +0100
Subject: [PATCH 225/247] Bump golang.org/x/oauth2 from 0.25.0 to 0.26.0
 (#2322)

Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from
0.25.0 to 0.26.0.
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/golang/oauth2/commit/b9c813be7d0ec3262d46deb8677ba5cda93d95ec"><code>b9c813b</code></a>
google: add warning about externally-provided credentials</li>
<li>See full diff in <a
href="https://github.com/golang/oauth2/compare/v0.25.0...v0.26.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/oauth2&package-manager=go_modules&previous-version=0.25.0&new-version=0.26.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index db3261beb..665e258bb 100644
--- a/go.mod
+++ b/go.mod
@@ -28,7 +28,7 @@ require (
 	github.com/wI2L/jsondiff v0.6.1 // MIT
 	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
 	golang.org/x/mod v0.22.0
-	golang.org/x/oauth2 v0.25.0
+	golang.org/x/oauth2 v0.26.0
 	golang.org/x/sync v0.10.0
 	golang.org/x/term v0.28.0
 	golang.org/x/text v0.21.0
diff --git a/go.sum b/go.sum
index d6102a0d1..87118cf0c 100644
--- a/go.sum
+++ b/go.sum
@@ -210,8 +210,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70=
-golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
+golang.org/x/oauth2 v0.26.0 h1:afQXWNNaeC4nvZ0Ed9XvCCzXM6UHJG7iCg0W4fPqSBE=
+golang.org/x/oauth2 v0.26.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

From f7a45d0c7ef2e3322509e3c214d7927bba61afa9 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Mon, 10 Feb 2025 14:06:02 +0000
Subject: [PATCH 226/247] Upgrade to TF provider 1.65.1 (#2328)

## Changes
Upgrade to TF provider 1.65.1

Notable changes:
- Now it's possible to use `run_as` field in `pipelines` definition
- Added support for `performance_target` for `jobs`
---
 acceptance/terraform/main.tf                  |   2 +-
 acceptance/terraform/output.txt               |   6 +-
 .../terraform/tfdyn/convert_pipeline_test.go  |   3 +
 bundle/internal/tf/codegen/README.md          |   3 +
 bundle/internal/tf/codegen/schema/version.go  |   2 +-
 .../internal/tf/schema/data_source_catalog.go |   1 -
 ...shboard_embedding_access_policy_setting.go |  14 ++
 ...oard_embedding_approved_domains_setting.go |  14 ++
 .../schema/resource_custom_app_integration.go |  25 +-
 bundle/internal/tf/schema/resource_job.go     |   1 +
 .../internal/tf/schema/resource_pipeline.go   |   6 +
 bundle/internal/tf/schema/resources.go        | 214 +++++++++---------
 bundle/internal/tf/schema/root.go             |   2 +-
 13 files changed, 169 insertions(+), 124 deletions(-)
 create mode 100644 bundle/internal/tf/schema/resource_aibi_dashboard_embedding_access_policy_setting.go
 create mode 100644 bundle/internal/tf/schema/resource_aibi_dashboard_embedding_approved_domains_setting.go

diff --git a/acceptance/terraform/main.tf b/acceptance/terraform/main.tf
index 93f665ff4..674b41a3c 100644
--- a/acceptance/terraform/main.tf
+++ b/acceptance/terraform/main.tf
@@ -2,7 +2,7 @@ terraform {
   required_providers {
     databricks = {
       source  = "databricks/databricks"
-      version = "1.64.1"
+      version = "1.65.1"
     }
   }
 
diff --git a/acceptance/terraform/output.txt b/acceptance/terraform/output.txt
index 6bdc809f6..851785827 100644
--- a/acceptance/terraform/output.txt
+++ b/acceptance/terraform/output.txt
@@ -4,9 +4,9 @@
 Initializing the backend...
 
 Initializing provider plugins...
-- Finding databricks/databricks versions matching "1.64.1"...
-- Installing databricks/databricks v1.64.1...
-- Installed databricks/databricks v1.64.1 (unauthenticated)
+- Finding databricks/databricks versions matching "1.65.1"...
+- Installing databricks/databricks v1.65.1...
+- Installed databricks/databricks v1.65.1 (unauthenticated)
 
 Terraform has created a lock file .terraform.lock.hcl to record the provider
 selections it made above. Include this file in your version control repository
diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go b/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go
index d8de55bf0..63d023c43 100644
--- a/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go
+++ b/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go
@@ -122,6 +122,9 @@ func TestConvertPipeline(t *testing.T) {
 				"num_workers": int64(1),
 			},
 		},
+		"run_as": map[string]any{
+			"user_name": "foo@bar.com",
+		},
 	}, out.Pipeline["my_pipeline"])
 
 	// Assert equality on the permissions
diff --git a/bundle/internal/tf/codegen/README.md b/bundle/internal/tf/codegen/README.md
index b1f8a33a8..968bf29ed 100644
--- a/bundle/internal/tf/codegen/README.md
+++ b/bundle/internal/tf/codegen/README.md
@@ -19,3 +19,6 @@ How to regenerate Go structs from an updated terraform provider?
 2. Delete `./tmp` if it exists
 3. Run `go run .`
 4. Run `gofmt -s -w ../schema`
+5. Go back to the root of the repo.
+6. Update `/acceptance/terraform/main.tf` file to use new version of TF provider
+7. Run `go test ./acceptance -v -update -run TestAccept/terraform` to update test output with a new version of TF provider
diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go
index 393afd6ed..46548f3e8 100644
--- a/bundle/internal/tf/codegen/schema/version.go
+++ b/bundle/internal/tf/codegen/schema/version.go
@@ -1,3 +1,3 @@
 package schema
 
-const ProviderVersion = "1.64.1"
+const ProviderVersion = "1.65.1"
diff --git a/bundle/internal/tf/schema/data_source_catalog.go b/bundle/internal/tf/schema/data_source_catalog.go
index 6f9237cfa..4b8c6df97 100644
--- a/bundle/internal/tf/schema/data_source_catalog.go
+++ b/bundle/internal/tf/schema/data_source_catalog.go
@@ -28,7 +28,6 @@ type DataSourceCatalogCatalogInfo struct {
 	Owner                               string                                                           `json:"owner,omitempty"`
 	Properties                          map[string]string                                                `json:"properties,omitempty"`
 	ProviderName                        string                                                           `json:"provider_name,omitempty"`
-	SecurableKind                       string                                                           `json:"securable_kind,omitempty"`
 	SecurableType                       string                                                           `json:"securable_type,omitempty"`
 	ShareName                           string                                                           `json:"share_name,omitempty"`
 	StorageLocation                     string                                                           `json:"storage_location,omitempty"`
diff --git a/bundle/internal/tf/schema/resource_aibi_dashboard_embedding_access_policy_setting.go b/bundle/internal/tf/schema/resource_aibi_dashboard_embedding_access_policy_setting.go
new file mode 100644
index 000000000..d816b235d
--- /dev/null
+++ b/bundle/internal/tf/schema/resource_aibi_dashboard_embedding_access_policy_setting.go
@@ -0,0 +1,14 @@
+// Generated from Databricks Terraform provider schema. DO NOT EDIT.
+
+package schema
+
+type ResourceAibiDashboardEmbeddingAccessPolicySettingAibiDashboardEmbeddingAccessPolicy struct {
+	AccessPolicyType string `json:"access_policy_type"`
+}
+
+type ResourceAibiDashboardEmbeddingAccessPolicySetting struct {
+	Etag                               string                                                                               `json:"etag,omitempty"`
+	Id                                 string                                                                               `json:"id,omitempty"`
+	SettingName                        string                                                                               `json:"setting_name,omitempty"`
+	AibiDashboardEmbeddingAccessPolicy *ResourceAibiDashboardEmbeddingAccessPolicySettingAibiDashboardEmbeddingAccessPolicy `json:"aibi_dashboard_embedding_access_policy,omitempty"`
+}
diff --git a/bundle/internal/tf/schema/resource_aibi_dashboard_embedding_approved_domains_setting.go b/bundle/internal/tf/schema/resource_aibi_dashboard_embedding_approved_domains_setting.go
new file mode 100644
index 000000000..690b334cd
--- /dev/null
+++ b/bundle/internal/tf/schema/resource_aibi_dashboard_embedding_approved_domains_setting.go
@@ -0,0 +1,14 @@
+// Generated from Databricks Terraform provider schema. DO NOT EDIT.
+
+package schema
+
+type ResourceAibiDashboardEmbeddingApprovedDomainsSettingAibiDashboardEmbeddingApprovedDomains struct {
+	ApprovedDomains []string `json:"approved_domains"`
+}
+
+type ResourceAibiDashboardEmbeddingApprovedDomainsSetting struct {
+	Etag                                  string                                                                                     `json:"etag,omitempty"`
+	Id                                    string                                                                                     `json:"id,omitempty"`
+	SettingName                           string                                                                                     `json:"setting_name,omitempty"`
+	AibiDashboardEmbeddingApprovedDomains *ResourceAibiDashboardEmbeddingApprovedDomainsSettingAibiDashboardEmbeddingApprovedDomains `json:"aibi_dashboard_embedding_approved_domains,omitempty"`
+}
diff --git a/bundle/internal/tf/schema/resource_custom_app_integration.go b/bundle/internal/tf/schema/resource_custom_app_integration.go
index e89eb7fe5..0a964f6ab 100644
--- a/bundle/internal/tf/schema/resource_custom_app_integration.go
+++ b/bundle/internal/tf/schema/resource_custom_app_integration.go
@@ -8,16 +8,17 @@ type ResourceCustomAppIntegrationTokenAccessPolicy struct {
 }
 
 type ResourceCustomAppIntegration struct {
-	ClientId          string                                         `json:"client_id,omitempty"`
-	ClientSecret      string                                         `json:"client_secret,omitempty"`
-	Confidential      bool                                           `json:"confidential,omitempty"`
-	CreateTime        string                                         `json:"create_time,omitempty"`
-	CreatedBy         int                                            `json:"created_by,omitempty"`
-	CreatorUsername   string                                         `json:"creator_username,omitempty"`
-	Id                string                                         `json:"id,omitempty"`
-	IntegrationId     string                                         `json:"integration_id,omitempty"`
-	Name              string                                         `json:"name,omitempty"`
-	RedirectUrls      []string                                       `json:"redirect_urls,omitempty"`
-	Scopes            []string                                       `json:"scopes,omitempty"`
-	TokenAccessPolicy *ResourceCustomAppIntegrationTokenAccessPolicy `json:"token_access_policy,omitempty"`
+	ClientId             string                                         `json:"client_id,omitempty"`
+	ClientSecret         string                                         `json:"client_secret,omitempty"`
+	Confidential         bool                                           `json:"confidential,omitempty"`
+	CreateTime           string                                         `json:"create_time,omitempty"`
+	CreatedBy            int                                            `json:"created_by,omitempty"`
+	CreatorUsername      string                                         `json:"creator_username,omitempty"`
+	Id                   string                                         `json:"id,omitempty"`
+	IntegrationId        string                                         `json:"integration_id,omitempty"`
+	Name                 string                                         `json:"name,omitempty"`
+	RedirectUrls         []string                                       `json:"redirect_urls,omitempty"`
+	Scopes               []string                                       `json:"scopes,omitempty"`
+	UserAuthorizedScopes []string                                       `json:"user_authorized_scopes,omitempty"`
+	TokenAccessPolicy    *ResourceCustomAppIntegrationTokenAccessPolicy `json:"token_access_policy,omitempty"`
 }
diff --git a/bundle/internal/tf/schema/resource_job.go b/bundle/internal/tf/schema/resource_job.go
index da277b5c1..2c27f0be7 100644
--- a/bundle/internal/tf/schema/resource_job.go
+++ b/bundle/internal/tf/schema/resource_job.go
@@ -1489,6 +1489,7 @@ type ResourceJob struct {
 	MaxRetries             int                              `json:"max_retries,omitempty"`
 	MinRetryIntervalMillis int                              `json:"min_retry_interval_millis,omitempty"`
 	Name                   string                           `json:"name,omitempty"`
+	PerformanceTarget      string                           `json:"performance_target,omitempty"`
 	RetryOnTimeout         bool                             `json:"retry_on_timeout,omitempty"`
 	Tags                   map[string]string                `json:"tags,omitempty"`
 	TimeoutSeconds         int                              `json:"timeout_seconds,omitempty"`
diff --git a/bundle/internal/tf/schema/resource_pipeline.go b/bundle/internal/tf/schema/resource_pipeline.go
index ebdb85027..8e260e65c 100644
--- a/bundle/internal/tf/schema/resource_pipeline.go
+++ b/bundle/internal/tf/schema/resource_pipeline.go
@@ -249,6 +249,11 @@ type ResourcePipelineRestartWindow struct {
 	TimeZoneId string   `json:"time_zone_id,omitempty"`
 }
 
+type ResourcePipelineRunAs struct {
+	ServicePrincipalName string `json:"service_principal_name,omitempty"`
+	UserName             string `json:"user_name,omitempty"`
+}
+
 type ResourcePipelineTriggerCron struct {
 	QuartzCronSchedule string `json:"quartz_cron_schedule,omitempty"`
 	TimezoneId         string `json:"timezone_id,omitempty"`
@@ -296,5 +301,6 @@ type ResourcePipeline struct {
 	Library              []ResourcePipelineLibrary            `json:"library,omitempty"`
 	Notification         []ResourcePipelineNotification       `json:"notification,omitempty"`
 	RestartWindow        *ResourcePipelineRestartWindow       `json:"restart_window,omitempty"`
+	RunAs                *ResourcePipelineRunAs               `json:"run_as,omitempty"`
 	Trigger              *ResourcePipelineTrigger             `json:"trigger,omitempty"`
 }
diff --git a/bundle/internal/tf/schema/resources.go b/bundle/internal/tf/schema/resources.go
index b57c2711a..c6eaa5b21 100644
--- a/bundle/internal/tf/schema/resources.go
+++ b/bundle/internal/tf/schema/resources.go
@@ -3,115 +3,119 @@
 package schema
 
 type Resources struct {
-	AccessControlRuleSet                       map[string]any `json:"databricks_access_control_rule_set,omitempty"`
-	Alert                                      map[string]any `json:"databricks_alert,omitempty"`
-	App                                        map[string]any `json:"databricks_app,omitempty"`
-	ArtifactAllowlist                          map[string]any `json:"databricks_artifact_allowlist,omitempty"`
-	AutomaticClusterUpdateWorkspaceSetting     map[string]any `json:"databricks_automatic_cluster_update_workspace_setting,omitempty"`
-	AwsS3Mount                                 map[string]any `json:"databricks_aws_s3_mount,omitempty"`
-	AzureAdlsGen1Mount                         map[string]any `json:"databricks_azure_adls_gen1_mount,omitempty"`
-	AzureAdlsGen2Mount                         map[string]any `json:"databricks_azure_adls_gen2_mount,omitempty"`
-	AzureBlobMount                             map[string]any `json:"databricks_azure_blob_mount,omitempty"`
-	Budget                                     map[string]any `json:"databricks_budget,omitempty"`
-	Catalog                                    map[string]any `json:"databricks_catalog,omitempty"`
-	CatalogWorkspaceBinding                    map[string]any `json:"databricks_catalog_workspace_binding,omitempty"`
-	Cluster                                    map[string]any `json:"databricks_cluster,omitempty"`
-	ClusterPolicy                              map[string]any `json:"databricks_cluster_policy,omitempty"`
-	ComplianceSecurityProfileWorkspaceSetting  map[string]any `json:"databricks_compliance_security_profile_workspace_setting,omitempty"`
-	Connection                                 map[string]any `json:"databricks_connection,omitempty"`
-	Credential                                 map[string]any `json:"databricks_credential,omitempty"`
-	CustomAppIntegration                       map[string]any `json:"databricks_custom_app_integration,omitempty"`
-	Dashboard                                  map[string]any `json:"databricks_dashboard,omitempty"`
-	DbfsFile                                   map[string]any `json:"databricks_dbfs_file,omitempty"`
-	DefaultNamespaceSetting                    map[string]any `json:"databricks_default_namespace_setting,omitempty"`
-	Directory                                  map[string]any `json:"databricks_directory,omitempty"`
-	EnhancedSecurityMonitoringWorkspaceSetting map[string]any `json:"databricks_enhanced_security_monitoring_workspace_setting,omitempty"`
-	Entitlements                               map[string]any `json:"databricks_entitlements,omitempty"`
-	ExternalLocation                           map[string]any `json:"databricks_external_location,omitempty"`
-	File                                       map[string]any `json:"databricks_file,omitempty"`
-	GitCredential                              map[string]any `json:"databricks_git_credential,omitempty"`
-	GlobalInitScript                           map[string]any `json:"databricks_global_init_script,omitempty"`
-	Grant                                      map[string]any `json:"databricks_grant,omitempty"`
-	Grants                                     map[string]any `json:"databricks_grants,omitempty"`
-	Group                                      map[string]any `json:"databricks_group,omitempty"`
-	GroupInstanceProfile                       map[string]any `json:"databricks_group_instance_profile,omitempty"`
-	GroupMember                                map[string]any `json:"databricks_group_member,omitempty"`
-	GroupRole                                  map[string]any `json:"databricks_group_role,omitempty"`
-	InstancePool                               map[string]any `json:"databricks_instance_pool,omitempty"`
-	InstanceProfile                            map[string]any `json:"databricks_instance_profile,omitempty"`
-	IpAccessList                               map[string]any `json:"databricks_ip_access_list,omitempty"`
-	Job                                        map[string]any `json:"databricks_job,omitempty"`
-	LakehouseMonitor                           map[string]any `json:"databricks_lakehouse_monitor,omitempty"`
-	Library                                    map[string]any `json:"databricks_library,omitempty"`
-	Metastore                                  map[string]any `json:"databricks_metastore,omitempty"`
-	MetastoreAssignment                        map[string]any `json:"databricks_metastore_assignment,omitempty"`
-	MetastoreDataAccess                        map[string]any `json:"databricks_metastore_data_access,omitempty"`
-	MlflowExperiment                           map[string]any `json:"databricks_mlflow_experiment,omitempty"`
-	MlflowModel                                map[string]any `json:"databricks_mlflow_model,omitempty"`
-	MlflowWebhook                              map[string]any `json:"databricks_mlflow_webhook,omitempty"`
-	ModelServing                               map[string]any `json:"databricks_model_serving,omitempty"`
-	Mount                                      map[string]any `json:"databricks_mount,omitempty"`
-	MwsCredentials                             map[string]any `json:"databricks_mws_credentials,omitempty"`
-	MwsCustomerManagedKeys                     map[string]any `json:"databricks_mws_customer_managed_keys,omitempty"`
-	MwsLogDelivery                             map[string]any `json:"databricks_mws_log_delivery,omitempty"`
-	MwsNccBinding                              map[string]any `json:"databricks_mws_ncc_binding,omitempty"`
-	MwsNccPrivateEndpointRule                  map[string]any `json:"databricks_mws_ncc_private_endpoint_rule,omitempty"`
-	MwsNetworkConnectivityConfig               map[string]any `json:"databricks_mws_network_connectivity_config,omitempty"`
-	MwsNetworks                                map[string]any `json:"databricks_mws_networks,omitempty"`
-	MwsPermissionAssignment                    map[string]any `json:"databricks_mws_permission_assignment,omitempty"`
-	MwsPrivateAccessSettings                   map[string]any `json:"databricks_mws_private_access_settings,omitempty"`
-	MwsStorageConfigurations                   map[string]any `json:"databricks_mws_storage_configurations,omitempty"`
-	MwsVpcEndpoint                             map[string]any `json:"databricks_mws_vpc_endpoint,omitempty"`
-	MwsWorkspaces                              map[string]any `json:"databricks_mws_workspaces,omitempty"`
-	Notebook                                   map[string]any `json:"databricks_notebook,omitempty"`
-	NotificationDestination                    map[string]any `json:"databricks_notification_destination,omitempty"`
-	OboToken                                   map[string]any `json:"databricks_obo_token,omitempty"`
-	OnlineTable                                map[string]any `json:"databricks_online_table,omitempty"`
-	PermissionAssignment                       map[string]any `json:"databricks_permission_assignment,omitempty"`
-	Permissions                                map[string]any `json:"databricks_permissions,omitempty"`
-	Pipeline                                   map[string]any `json:"databricks_pipeline,omitempty"`
-	Provider                                   map[string]any `json:"databricks_provider,omitempty"`
-	QualityMonitor                             map[string]any `json:"databricks_quality_monitor,omitempty"`
-	Query                                      map[string]any `json:"databricks_query,omitempty"`
-	Recipient                                  map[string]any `json:"databricks_recipient,omitempty"`
-	RegisteredModel                            map[string]any `json:"databricks_registered_model,omitempty"`
-	Repo                                       map[string]any `json:"databricks_repo,omitempty"`
-	RestrictWorkspaceAdminsSetting             map[string]any `json:"databricks_restrict_workspace_admins_setting,omitempty"`
-	Schema                                     map[string]any `json:"databricks_schema,omitempty"`
-	Secret                                     map[string]any `json:"databricks_secret,omitempty"`
-	SecretAcl                                  map[string]any `json:"databricks_secret_acl,omitempty"`
-	SecretScope                                map[string]any `json:"databricks_secret_scope,omitempty"`
-	ServicePrincipal                           map[string]any `json:"databricks_service_principal,omitempty"`
-	ServicePrincipalRole                       map[string]any `json:"databricks_service_principal_role,omitempty"`
-	ServicePrincipalSecret                     map[string]any `json:"databricks_service_principal_secret,omitempty"`
-	Share                                      map[string]any `json:"databricks_share,omitempty"`
-	SqlAlert                                   map[string]any `json:"databricks_sql_alert,omitempty"`
-	SqlDashboard                               map[string]any `json:"databricks_sql_dashboard,omitempty"`
-	SqlEndpoint                                map[string]any `json:"databricks_sql_endpoint,omitempty"`
-	SqlGlobalConfig                            map[string]any `json:"databricks_sql_global_config,omitempty"`
-	SqlPermissions                             map[string]any `json:"databricks_sql_permissions,omitempty"`
-	SqlQuery                                   map[string]any `json:"databricks_sql_query,omitempty"`
-	SqlTable                                   map[string]any `json:"databricks_sql_table,omitempty"`
-	SqlVisualization                           map[string]any `json:"databricks_sql_visualization,omitempty"`
-	SqlWidget                                  map[string]any `json:"databricks_sql_widget,omitempty"`
-	StorageCredential                          map[string]any `json:"databricks_storage_credential,omitempty"`
-	SystemSchema                               map[string]any `json:"databricks_system_schema,omitempty"`
-	Table                                      map[string]any `json:"databricks_table,omitempty"`
-	Token                                      map[string]any `json:"databricks_token,omitempty"`
-	User                                       map[string]any `json:"databricks_user,omitempty"`
-	UserInstanceProfile                        map[string]any `json:"databricks_user_instance_profile,omitempty"`
-	UserRole                                   map[string]any `json:"databricks_user_role,omitempty"`
-	VectorSearchEndpoint                       map[string]any `json:"databricks_vector_search_endpoint,omitempty"`
-	VectorSearchIndex                          map[string]any `json:"databricks_vector_search_index,omitempty"`
-	Volume                                     map[string]any `json:"databricks_volume,omitempty"`
-	WorkspaceBinding                           map[string]any `json:"databricks_workspace_binding,omitempty"`
-	WorkspaceConf                              map[string]any `json:"databricks_workspace_conf,omitempty"`
-	WorkspaceFile                              map[string]any `json:"databricks_workspace_file,omitempty"`
+	AccessControlRuleSet                         map[string]any `json:"databricks_access_control_rule_set,omitempty"`
+	AibiDashboardEmbeddingAccessPolicySetting    map[string]any `json:"databricks_aibi_dashboard_embedding_access_policy_setting,omitempty"`
+	AibiDashboardEmbeddingApprovedDomainsSetting map[string]any `json:"databricks_aibi_dashboard_embedding_approved_domains_setting,omitempty"`
+	Alert                                        map[string]any `json:"databricks_alert,omitempty"`
+	App                                          map[string]any `json:"databricks_app,omitempty"`
+	ArtifactAllowlist                            map[string]any `json:"databricks_artifact_allowlist,omitempty"`
+	AutomaticClusterUpdateWorkspaceSetting       map[string]any `json:"databricks_automatic_cluster_update_workspace_setting,omitempty"`
+	AwsS3Mount                                   map[string]any `json:"databricks_aws_s3_mount,omitempty"`
+	AzureAdlsGen1Mount                           map[string]any `json:"databricks_azure_adls_gen1_mount,omitempty"`
+	AzureAdlsGen2Mount                           map[string]any `json:"databricks_azure_adls_gen2_mount,omitempty"`
+	AzureBlobMount                               map[string]any `json:"databricks_azure_blob_mount,omitempty"`
+	Budget                                       map[string]any `json:"databricks_budget,omitempty"`
+	Catalog                                      map[string]any `json:"databricks_catalog,omitempty"`
+	CatalogWorkspaceBinding                      map[string]any `json:"databricks_catalog_workspace_binding,omitempty"`
+	Cluster                                      map[string]any `json:"databricks_cluster,omitempty"`
+	ClusterPolicy                                map[string]any `json:"databricks_cluster_policy,omitempty"`
+	ComplianceSecurityProfileWorkspaceSetting    map[string]any `json:"databricks_compliance_security_profile_workspace_setting,omitempty"`
+	Connection                                   map[string]any `json:"databricks_connection,omitempty"`
+	Credential                                   map[string]any `json:"databricks_credential,omitempty"`
+	CustomAppIntegration                         map[string]any `json:"databricks_custom_app_integration,omitempty"`
+	Dashboard                                    map[string]any `json:"databricks_dashboard,omitempty"`
+	DbfsFile                                     map[string]any `json:"databricks_dbfs_file,omitempty"`
+	DefaultNamespaceSetting                      map[string]any `json:"databricks_default_namespace_setting,omitempty"`
+	Directory                                    map[string]any `json:"databricks_directory,omitempty"`
+	EnhancedSecurityMonitoringWorkspaceSetting   map[string]any `json:"databricks_enhanced_security_monitoring_workspace_setting,omitempty"`
+	Entitlements                                 map[string]any `json:"databricks_entitlements,omitempty"`
+	ExternalLocation                             map[string]any `json:"databricks_external_location,omitempty"`
+	File                                         map[string]any `json:"databricks_file,omitempty"`
+	GitCredential                                map[string]any `json:"databricks_git_credential,omitempty"`
+	GlobalInitScript                             map[string]any `json:"databricks_global_init_script,omitempty"`
+	Grant                                        map[string]any `json:"databricks_grant,omitempty"`
+	Grants                                       map[string]any `json:"databricks_grants,omitempty"`
+	Group                                        map[string]any `json:"databricks_group,omitempty"`
+	GroupInstanceProfile                         map[string]any `json:"databricks_group_instance_profile,omitempty"`
+	GroupMember                                  map[string]any `json:"databricks_group_member,omitempty"`
+	GroupRole                                    map[string]any `json:"databricks_group_role,omitempty"`
+	InstancePool                                 map[string]any `json:"databricks_instance_pool,omitempty"`
+	InstanceProfile                              map[string]any `json:"databricks_instance_profile,omitempty"`
+	IpAccessList                                 map[string]any `json:"databricks_ip_access_list,omitempty"`
+	Job                                          map[string]any `json:"databricks_job,omitempty"`
+	LakehouseMonitor                             map[string]any `json:"databricks_lakehouse_monitor,omitempty"`
+	Library                                      map[string]any `json:"databricks_library,omitempty"`
+	Metastore                                    map[string]any `json:"databricks_metastore,omitempty"`
+	MetastoreAssignment                          map[string]any `json:"databricks_metastore_assignment,omitempty"`
+	MetastoreDataAccess                          map[string]any `json:"databricks_metastore_data_access,omitempty"`
+	MlflowExperiment                             map[string]any `json:"databricks_mlflow_experiment,omitempty"`
+	MlflowModel                                  map[string]any `json:"databricks_mlflow_model,omitempty"`
+	MlflowWebhook                                map[string]any `json:"databricks_mlflow_webhook,omitempty"`
+	ModelServing                                 map[string]any `json:"databricks_model_serving,omitempty"`
+	Mount                                        map[string]any `json:"databricks_mount,omitempty"`
+	MwsCredentials                               map[string]any `json:"databricks_mws_credentials,omitempty"`
+	MwsCustomerManagedKeys                       map[string]any `json:"databricks_mws_customer_managed_keys,omitempty"`
+	MwsLogDelivery                               map[string]any `json:"databricks_mws_log_delivery,omitempty"`
+	MwsNccBinding                                map[string]any `json:"databricks_mws_ncc_binding,omitempty"`
+	MwsNccPrivateEndpointRule                    map[string]any `json:"databricks_mws_ncc_private_endpoint_rule,omitempty"`
+	MwsNetworkConnectivityConfig                 map[string]any `json:"databricks_mws_network_connectivity_config,omitempty"`
+	MwsNetworks                                  map[string]any `json:"databricks_mws_networks,omitempty"`
+	MwsPermissionAssignment                      map[string]any `json:"databricks_mws_permission_assignment,omitempty"`
+	MwsPrivateAccessSettings                     map[string]any `json:"databricks_mws_private_access_settings,omitempty"`
+	MwsStorageConfigurations                     map[string]any `json:"databricks_mws_storage_configurations,omitempty"`
+	MwsVpcEndpoint                               map[string]any `json:"databricks_mws_vpc_endpoint,omitempty"`
+	MwsWorkspaces                                map[string]any `json:"databricks_mws_workspaces,omitempty"`
+	Notebook                                     map[string]any `json:"databricks_notebook,omitempty"`
+	NotificationDestination                      map[string]any `json:"databricks_notification_destination,omitempty"`
+	OboToken                                     map[string]any `json:"databricks_obo_token,omitempty"`
+	OnlineTable                                  map[string]any `json:"databricks_online_table,omitempty"`
+	PermissionAssignment                         map[string]any `json:"databricks_permission_assignment,omitempty"`
+	Permissions                                  map[string]any `json:"databricks_permissions,omitempty"`
+	Pipeline                                     map[string]any `json:"databricks_pipeline,omitempty"`
+	Provider                                     map[string]any `json:"databricks_provider,omitempty"`
+	QualityMonitor                               map[string]any `json:"databricks_quality_monitor,omitempty"`
+	Query                                        map[string]any `json:"databricks_query,omitempty"`
+	Recipient                                    map[string]any `json:"databricks_recipient,omitempty"`
+	RegisteredModel                              map[string]any `json:"databricks_registered_model,omitempty"`
+	Repo                                         map[string]any `json:"databricks_repo,omitempty"`
+	RestrictWorkspaceAdminsSetting               map[string]any `json:"databricks_restrict_workspace_admins_setting,omitempty"`
+	Schema                                       map[string]any `json:"databricks_schema,omitempty"`
+	Secret                                       map[string]any `json:"databricks_secret,omitempty"`
+	SecretAcl                                    map[string]any `json:"databricks_secret_acl,omitempty"`
+	SecretScope                                  map[string]any `json:"databricks_secret_scope,omitempty"`
+	ServicePrincipal                             map[string]any `json:"databricks_service_principal,omitempty"`
+	ServicePrincipalRole                         map[string]any `json:"databricks_service_principal_role,omitempty"`
+	ServicePrincipalSecret                       map[string]any `json:"databricks_service_principal_secret,omitempty"`
+	Share                                        map[string]any `json:"databricks_share,omitempty"`
+	SqlAlert                                     map[string]any `json:"databricks_sql_alert,omitempty"`
+	SqlDashboard                                 map[string]any `json:"databricks_sql_dashboard,omitempty"`
+	SqlEndpoint                                  map[string]any `json:"databricks_sql_endpoint,omitempty"`
+	SqlGlobalConfig                              map[string]any `json:"databricks_sql_global_config,omitempty"`
+	SqlPermissions                               map[string]any `json:"databricks_sql_permissions,omitempty"`
+	SqlQuery                                     map[string]any `json:"databricks_sql_query,omitempty"`
+	SqlTable                                     map[string]any `json:"databricks_sql_table,omitempty"`
+	SqlVisualization                             map[string]any `json:"databricks_sql_visualization,omitempty"`
+	SqlWidget                                    map[string]any `json:"databricks_sql_widget,omitempty"`
+	StorageCredential                            map[string]any `json:"databricks_storage_credential,omitempty"`
+	SystemSchema                                 map[string]any `json:"databricks_system_schema,omitempty"`
+	Table                                        map[string]any `json:"databricks_table,omitempty"`
+	Token                                        map[string]any `json:"databricks_token,omitempty"`
+	User                                         map[string]any `json:"databricks_user,omitempty"`
+	UserInstanceProfile                          map[string]any `json:"databricks_user_instance_profile,omitempty"`
+	UserRole                                     map[string]any `json:"databricks_user_role,omitempty"`
+	VectorSearchEndpoint                         map[string]any `json:"databricks_vector_search_endpoint,omitempty"`
+	VectorSearchIndex                            map[string]any `json:"databricks_vector_search_index,omitempty"`
+	Volume                                       map[string]any `json:"databricks_volume,omitempty"`
+	WorkspaceBinding                             map[string]any `json:"databricks_workspace_binding,omitempty"`
+	WorkspaceConf                                map[string]any `json:"databricks_workspace_conf,omitempty"`
+	WorkspaceFile                                map[string]any `json:"databricks_workspace_file,omitempty"`
 }
 
 func NewResources() *Resources {
 	return &Resources{
-		AccessControlRuleSet:                   make(map[string]any),
+		AccessControlRuleSet:                         make(map[string]any),
+		AibiDashboardEmbeddingAccessPolicySetting:    make(map[string]any),
+		AibiDashboardEmbeddingApprovedDomainsSetting: make(map[string]any),
 		Alert:                                  make(map[string]any),
 		App:                                    make(map[string]any),
 		ArtifactAllowlist:                      make(map[string]any),
diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go
index 2ac852355..816e8e6aa 100644
--- a/bundle/internal/tf/schema/root.go
+++ b/bundle/internal/tf/schema/root.go
@@ -21,7 +21,7 @@ type Root struct {
 
 const ProviderHost = "registry.terraform.io"
 const ProviderSource = "databricks/databricks"
-const ProviderVersion = "1.64.1"
+const ProviderVersion = "1.65.1"
 
 func NewRoot() *Root {
 	return &Root{

From f6c50a631801f0a08a549b358eb129283d1e6bc0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 10 Feb 2025 15:49:52 +0100
Subject: [PATCH 227/247] Bump golang.org/x/term from 0.28.0 to 0.29.0 (#2325)

Bumps [golang.org/x/term](https://github.com/golang/term) from 0.28.0 to
0.29.0.
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/golang/term/commit/743b2709ab25357d30ce1eac4a840eb0b7deb1bf"><code>743b270</code></a>
go.mod: update golang.org/x dependencies</li>
<li>See full diff in <a
href="https://github.com/golang/term/compare/v0.28.0...v0.29.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/term&package-manager=go_modules&previous-version=0.28.0&new-version=0.29.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 4 ++--
 go.sum | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/go.mod b/go.mod
index 665e258bb..11bc1f87f 100644
--- a/go.mod
+++ b/go.mod
@@ -30,7 +30,7 @@ require (
 	golang.org/x/mod v0.22.0
 	golang.org/x/oauth2 v0.26.0
 	golang.org/x/sync v0.10.0
-	golang.org/x/term v0.28.0
+	golang.org/x/term v0.29.0
 	golang.org/x/text v0.21.0
 	gopkg.in/ini.v1 v1.67.0 // Apache 2.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -71,7 +71,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	golang.org/x/crypto v0.31.0 // indirect
 	golang.org/x/net v0.33.0 // indirect
-	golang.org/x/sys v0.29.0 // indirect
+	golang.org/x/sys v0.30.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	google.golang.org/api v0.182.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect
diff --git a/go.sum b/go.sum
index 87118cf0c..523a44abd 100644
--- a/go.sum
+++ b/go.sum
@@ -227,10 +227,10 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
-golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
-golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
+golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
+golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=

From 6f3dbaec4ccb97cb83b4807070da43662457f262 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 10 Feb 2025 16:13:38 +0100
Subject: [PATCH 228/247] Bump golang.org/x/text from 0.21.0 to 0.22.0 (#2323)

Bumps [golang.org/x/text](https://github.com/golang/text) from 0.21.0 to
0.22.0.
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/golang/text/commit/3b64043c9e8fa8cd61a019df17dc729630915fa9"><code>3b64043</code></a>
go.mod: update golang.org/x dependencies</li>
<li><a
href="https://github.com/golang/text/commit/1e59086680cc4c90cd7e9a2f5fcf98043df4396c"><code>1e59086</code></a>
message/pipeline: add two Unalias calls</li>
<li>See full diff in <a
href="https://github.com/golang/text/compare/v0.21.0...v0.22.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/text&package-manager=go_modules&previous-version=0.21.0&new-version=0.22.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 4 ++--
 go.sum | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/go.mod b/go.mod
index 11bc1f87f..16242a09e 100644
--- a/go.mod
+++ b/go.mod
@@ -29,9 +29,9 @@ require (
 	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
 	golang.org/x/mod v0.22.0
 	golang.org/x/oauth2 v0.26.0
-	golang.org/x/sync v0.10.0
+	golang.org/x/sync v0.11.0
 	golang.org/x/term v0.29.0
-	golang.org/x/text v0.21.0
+	golang.org/x/text v0.22.0
 	gopkg.in/ini.v1 v1.67.0 // Apache 2.0
 	gopkg.in/yaml.v3 v3.0.1
 )
diff --git a/go.sum b/go.sum
index 523a44abd..13e51beec 100644
--- a/go.sum
+++ b/go.sum
@@ -215,8 +215,8 @@ golang.org/x/oauth2 v0.26.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
-golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
+golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -233,8 +233,8 @@ golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
 golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
+golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

From e81ec4ee237bd6066f1d45a0c3a789c22e626104 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 10 Feb 2025 17:13:49 +0100
Subject: [PATCH 229/247] Bump golang.org/x/mod from 0.22.0 to 0.23.0 (#2324)

Bumps [golang.org/x/mod](https://github.com/golang/mod) from 0.22.0 to
0.23.0.
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/golang/mod/commit/52289f1fa75a8da0eb82d369cf5fda65fd6147b9"><code>52289f1</code></a>
modfile: fix trailing empty lines in require blocks</li>
<li>See full diff in <a
href="https://github.com/golang/mod/compare/v0.22.0...v0.23.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/mod&package-manager=go_modules&previous-version=0.22.0&new-version=0.23.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 16242a09e..c8b209edd 100644
--- a/go.mod
+++ b/go.mod
@@ -27,7 +27,7 @@ require (
 	github.com/stretchr/testify v1.10.0 // MIT
 	github.com/wI2L/jsondiff v0.6.1 // MIT
 	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
-	golang.org/x/mod v0.22.0
+	golang.org/x/mod v0.23.0
 	golang.org/x/oauth2 v0.26.0
 	golang.org/x/sync v0.11.0
 	golang.org/x/term v0.29.0
diff --git a/go.sum b/go.sum
index 13e51beec..0369fc2d9 100644
--- a/go.sum
+++ b/go.sum
@@ -199,8 +199,8 @@ golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
-golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
+golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
+golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

From f2096eddcc7e586216c36911430fa67bd40a427c Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 11 Feb 2025 10:38:53 +0100
Subject: [PATCH 230/247] acc: Do not show all replacements on every failure
 (#2331)

## Changes
- Only print replacements if VERBOSE_TEST flag is set.
- This is set on CI but not when you do "go test" or "make test".

Note, env var is used, so that it can be set in Makefile.

## Tests
Manually.
---
 Makefile                      | 4 ++--
 acceptance/acceptance_test.go | 7 ++++---
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index fb3936184..0c3860e29 100644
--- a/Makefile
+++ b/Makefile
@@ -28,7 +28,7 @@ test:
 
 cover:
 	rm -fr ./acceptance/build/cover/
-	CLI_GOCOVERDIR=build/cover ${GOTESTSUM_CMD} -- -coverprofile=coverage.txt ${PACKAGES}
+	VERBOSE_TEST=1 CLI_GOCOVERDIR=build/cover ${GOTESTSUM_CMD} -- -coverprofile=coverage.txt ${PACKAGES}
 	rm -fr ./acceptance/build/cover-merged/
 	mkdir -p acceptance/build/cover-merged/
 	go tool covdata merge -i $$(printf '%s,' acceptance/build/cover/* | sed 's/,$$//') -o acceptance/build/cover-merged/
@@ -61,6 +61,6 @@ integration: vendor
 	$(INTEGRATION)
 
 integration-short: vendor
-	$(INTEGRATION) -short
+	VERBOSE_TEST=1 $(INTEGRATION) -short
 
 .PHONY: lint tidy lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover docs
diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index b05b10f47..94db3232d 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -30,8 +30,9 @@ import (
 )
 
 var (
-	KeepTmp bool
-	NoRepl  bool
+	KeepTmp     bool
+	NoRepl      bool
+	VerboseTest bool = os.Getenv("VERBOSE_TEST") != ""
 )
 
 // In order to debug CLI running under acceptance test, set this to full subtest name, e.g. "bundle/variables/empty"
@@ -412,7 +413,7 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN
 		testutil.WriteFile(t, pathRef, valueNew)
 	}
 
-	if !equal && printedRepls != nil && !*printedRepls {
+	if VerboseTest && !equal && printedRepls != nil && !*printedRepls {
 		*printedRepls = true
 		var items []string
 		for _, item := range repls.Repls {

From 8d849fe868ed2807cb47d037323847a0b1bfc4b4 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 11 Feb 2025 11:37:48 +0100
Subject: [PATCH 231/247] acc: Disable custom server on CLOUD_ENV (#2332)

We're not using local server when CLOUD_ENV is enabled, no need to set
up a custom one.
---
 acceptance/acceptance_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 94db3232d..320948fda 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -262,7 +262,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 	// 2. The test is configured to record requests and assert on them. We need
 	//    a duplicate of the default server to record requests because the default
 	//    server otherwise is a shared resource.
-	if len(config.Server) > 0 || config.RecordRequests {
+	if cloudEnv == "" && (len(config.Server) > 0 || config.RecordRequests) {
 		server = testserver.New(t)
 		server.RecordRequests = config.RecordRequests
 		server.IncludeRequestHeaders = config.IncludeRequestHeaders

From 878fa803224a99176dcf2e4b4e94342389cb8a5a Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 11 Feb 2025 11:50:52 +0100
Subject: [PATCH 232/247] acc: Fix RecordRequests to support requests without
 body (#2333)

## Changes
Do not paste request body into output if it's not a valid JSON.

## Tests
While working on #2334 I found that if I try to record a test that calls
/api/2.0/preview/scim/v2/Me which has no request body, it crashes.
---
 libs/testserver/server.go | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/libs/testserver/server.go b/libs/testserver/server.go
index d5877e90e..577ef082c 100644
--- a/libs/testserver/server.go
+++ b/libs/testserver/server.go
@@ -34,7 +34,8 @@ type Request struct {
 	Headers http.Header `json:"headers,omitempty"`
 	Method  string      `json:"method"`
 	Path    string      `json:"path"`
-	Body    any         `json:"body"`
+	Body    any         `json:"body,omitempty"`
+	RawBody string      `json:"raw_body,omitempty"`
 }
 
 func New(t testutil.TestingT) *Server {
@@ -119,13 +120,19 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) {
 				}
 			}
 
-			s.Requests = append(s.Requests, Request{
+			req := Request{
 				Headers: headers,
 				Method:  r.Method,
 				Path:    r.URL.Path,
-				Body:    json.RawMessage(body),
-			})
+			}
 
+			if json.Valid(body) {
+				req.Body = json.RawMessage(body)
+			} else {
+				req.RawBody = string(body)
+			}
+
+			s.Requests = append(s.Requests, req)
 		}
 
 		w.Header().Set("Content-Type", "application/json")

From 272ce6130272c28f6c8e0a9113a3727b952282fe Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 11 Feb 2025 15:26:46 +0100
Subject: [PATCH 233/247] acc: Fix singleTest option to support forward slashes
 (#2336)

The filtering of tests needs to see forward slashes otherwise it is
OS-dependent.

I've also switched to filepath.ToSlash but it should be a no-op.
---
 acceptance/acceptance_test.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 320948fda..fce508498 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -180,8 +180,7 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	}
 
 	for _, dir := range testDirs {
-		testName := strings.ReplaceAll(dir, "\\", "/")
-		t.Run(testName, func(t *testing.T) {
+		t.Run(dir, func(t *testing.T) {
 			if !InprocessMode {
 				t.Parallel()
 			}
@@ -203,7 +202,8 @@ func getTests(t *testing.T) []string {
 		name := filepath.Base(path)
 		if name == EntryPointScript {
 			// Presence of 'script' marks a test case in this directory
-			testDirs = append(testDirs, filepath.Dir(path))
+			testName := filepath.ToSlash(filepath.Dir(path))
+			testDirs = append(testDirs, testName)
 		}
 		return nil
 	})

From 5d392acbef7873d3dcb54b314181381da89dda03 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 11 Feb 2025 16:03:41 +0100
Subject: [PATCH 234/247] acc: Allow mixing custom stubs with default server
 impl (#2334)

## Changes
- Currently if you define [[Server]] block, you disable the default
server implementation. With this change [[Server]] block takes
precedence over default server but default server remains.
- Switched mux implementation to
[gorilla/mux](https://github.com/gorilla/mux) -- unlike built-in it does
not panic if you set two handlers on the same part (instead the earliest
one wins). It also does not have any dependencies.
- Move acceptance/selftest into acceptance/selftest/basic and added
acceptance/selftest/server that demoes server override.
- Rewrite server set up to ensure that env vars and replacements are set
up correctly. Previously replacements for DATABRICKS_HOST referred to
default server, not to the custom server.
- Avoid calling CurrentUser.Me() in the local case. This allows
overriding /api/2.0/preview/scim/v2/Me, which we use in some tests (e.g.
bundle/templates-machinery/helpers-error). Previously the test passed
because CurrentUser.Me() was calling default server which is incorrect
but it happened to make the tests pass.
- The default server is now available on DATABRICKS_DEFAULT_HOST env
var.
- Rewrite "not found" handler in local test to handle error better (do
not raise http500 when header is already written).

## Tests
New acceptance test selftest/server specifically tests that both custom
and default handlers are available in a single test.
---
 NOTICE                                        |   4 +
 acceptance/acceptance_test.go                 | 123 ++++++++++--------
 acceptance/auth/bundle_and_profile/output.txt |   4 +-
 acceptance/auth/bundle_and_profile/test.toml  |   6 +-
 acceptance/cmd_server_test.go                 |   6 +-
 acceptance/selftest/{ => basic}/out.hello.txt |   0
 acceptance/selftest/{ => basic}/output.txt    |   0
 acceptance/selftest/{ => basic}/script        |   0
 acceptance/selftest/{ => basic}/test.toml     |   0
 acceptance/selftest/server/out.requests.txt   |   8 ++
 acceptance/selftest/server/output.txt         |  15 +++
 acceptance/selftest/server/script             |   2 +
 acceptance/selftest/server/test.toml          |  18 +++
 acceptance/server_test.go                     |  45 ++++---
 go.mod                                        |   1 +
 go.sum                                        |   2 +
 libs/testserver/server.go                     |  39 ++++--
 17 files changed, 179 insertions(+), 94 deletions(-)
 rename acceptance/selftest/{ => basic}/out.hello.txt (100%)
 rename acceptance/selftest/{ => basic}/output.txt (100%)
 rename acceptance/selftest/{ => basic}/script (100%)
 rename acceptance/selftest/{ => basic}/test.toml (100%)
 create mode 100644 acceptance/selftest/server/out.requests.txt
 create mode 100644 acceptance/selftest/server/output.txt
 create mode 100644 acceptance/selftest/server/script
 create mode 100644 acceptance/selftest/server/test.toml

diff --git a/NOTICE b/NOTICE
index 4331a2a32..0b1d2da04 100644
--- a/NOTICE
+++ b/NOTICE
@@ -114,3 +114,7 @@ dario.cat/mergo
 Copyright (c) 2013 Dario Castañé. All rights reserved.
 Copyright (c) 2012 The Go Authors. All rights reserved.
 https://github.com/darccio/mergo/blob/master/LICENSE
+
+https://github.com/gorilla/mux
+Copyright (c) 2023 The Gorilla Authors. All rights reserved.
+https://github.com/gorilla/mux/blob/main/LICENSE
diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index fce508498..117172f60 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -11,6 +11,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"slices"
 	"sort"
@@ -26,6 +27,7 @@ import (
 	"github.com/databricks/cli/libs/testdiff"
 	"github.com/databricks/cli/libs/testserver"
 	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/service/iam"
 	"github.com/stretchr/testify/require"
 )
 
@@ -72,7 +74,8 @@ func TestInprocessMode(t *testing.T) {
 	if InprocessMode {
 		t.Skip("Already tested by TestAccept")
 	}
-	require.Equal(t, 1, testAccept(t, true, "selftest"))
+	require.Equal(t, 1, testAccept(t, true, "selftest/basic"))
+	require.Equal(t, 1, testAccept(t, true, "selftest/server"))
 }
 
 func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
@@ -118,14 +121,12 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	uvCache := getUVDefaultCacheDir(t)
 	t.Setenv("UV_CACHE_DIR", uvCache)
 
-	ctx := context.Background()
 	cloudEnv := os.Getenv("CLOUD_ENV")
 
 	if cloudEnv == "" {
 		defaultServer := testserver.New(t)
 		AddHandlers(defaultServer)
-		// Redirect API access to local server:
-		t.Setenv("DATABRICKS_HOST", defaultServer.URL)
+		t.Setenv("DATABRICKS_DEFAULT_HOST", defaultServer.URL)
 
 		homeDir := t.TempDir()
 		// Do not read user's ~/.databrickscfg
@@ -148,27 +149,12 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	// do it last so that full paths match first:
 	repls.SetPath(buildDir, "[BUILD_DIR]")
 
-	var config databricks.Config
-	if cloudEnv == "" {
-		// use fake token for local tests
-		config = databricks.Config{Token: "dbapi1234"}
-	} else {
-		// non-local tests rely on environment variables
-		config = databricks.Config{}
-	}
-	workspaceClient, err := databricks.NewWorkspaceClient(&config)
-	require.NoError(t, err)
-
-	user, err := workspaceClient.CurrentUser.Me(ctx)
-	require.NoError(t, err)
-	require.NotNil(t, user)
-	testdiff.PrepareReplacementsUser(t, &repls, *user)
-	testdiff.PrepareReplacementsWorkspaceClient(t, &repls, workspaceClient)
-	testdiff.PrepareReplacementsUUID(t, &repls)
 	testdiff.PrepareReplacementsDevVersion(t, &repls)
 	testdiff.PrepareReplacementSdkVersion(t, &repls)
 	testdiff.PrepareReplacementsGoVersion(t, &repls)
 
+	repls.Repls = append(repls.Repls, testdiff.Replacement{Old: regexp.MustCompile("dbapi[0-9a-f]+"), New: "[DATABRICKS_TOKEN]"})
+
 	testDirs := getTests(t)
 	require.NotEmpty(t, testDirs)
 
@@ -239,7 +225,6 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 	}
 
 	repls.SetPathWithParents(tmpDir, "[TMPDIR]")
-	repls.Repls = append(repls.Repls, config.Repls...)
 
 	scriptContents := readMergedScriptContents(t, dir)
 	testutil.WriteFile(t, filepath.Join(tmpDir, EntryPointScript), scriptContents)
@@ -253,38 +238,79 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 	cmd := exec.Command(args[0], args[1:]...)
 	cmd.Env = os.Environ()
 
+	var workspaceClient *databricks.WorkspaceClient
+	var user iam.User
+
 	// Start a new server with a custom configuration if the acceptance test
 	// specifies a custom server stubs.
 	var server *testserver.Server
 
-	// Start a new server for this test if either:
-	// 1. A custom server spec is defined in the test configuration.
-	// 2. The test is configured to record requests and assert on them. We need
-	//    a duplicate of the default server to record requests because the default
-	//    server otherwise is a shared resource.
-	if cloudEnv == "" && (len(config.Server) > 0 || config.RecordRequests) {
-		server = testserver.New(t)
-		server.RecordRequests = config.RecordRequests
-		server.IncludeRequestHeaders = config.IncludeRequestHeaders
+	if cloudEnv == "" {
+		// Start a new server for this test if either:
+		// 1. A custom server spec is defined in the test configuration.
+		// 2. The test is configured to record requests and assert on them. We need
+		//    a duplicate of the default server to record requests because the default
+		//    server otherwise is a shared resource.
 
-		// If no custom server stubs are defined, add the default handlers.
-		if len(config.Server) == 0 {
+		databricksLocalHost := os.Getenv("DATABRICKS_DEFAULT_HOST")
+
+		if len(config.Server) > 0 || config.RecordRequests {
+			server = testserver.New(t)
+			server.RecordRequests = config.RecordRequests
+			server.IncludeRequestHeaders = config.IncludeRequestHeaders
+
+			for _, stub := range config.Server {
+				require.NotEmpty(t, stub.Pattern)
+				items := strings.Split(stub.Pattern, " ")
+				require.Len(t, items, 2)
+				server.Handle(items[0], items[1], func(fakeWorkspace *testserver.FakeWorkspace, req *http.Request) (any, int) {
+					statusCode := http.StatusOK
+					if stub.Response.StatusCode != 0 {
+						statusCode = stub.Response.StatusCode
+					}
+					return stub.Response.Body, statusCode
+				})
+			}
+
+			// The earliest handlers take precedence, add default handlers last
 			AddHandlers(server)
+			databricksLocalHost = server.URL
 		}
 
-		for _, stub := range config.Server {
-			require.NotEmpty(t, stub.Pattern)
-			server.Handle(stub.Pattern, func(fakeWorkspace *testserver.FakeWorkspace, req *http.Request) (any, int) {
-				statusCode := http.StatusOK
-				if stub.Response.StatusCode != 0 {
-					statusCode = stub.Response.StatusCode
-				}
-				return stub.Response.Body, statusCode
-			})
+		// Each local test should use a new token that will result into a new fake workspace,
+		// so that test don't interfere with each other.
+		tokenSuffix := strings.ReplaceAll(uuid.NewString(), "-", "")
+		config := databricks.Config{
+			Host:  databricksLocalHost,
+			Token: "dbapi" + tokenSuffix,
 		}
-		cmd.Env = append(cmd.Env, "DATABRICKS_HOST="+server.URL)
+		workspaceClient, err = databricks.NewWorkspaceClient(&config)
+		require.NoError(t, err)
+
+		cmd.Env = append(cmd.Env, "DATABRICKS_HOST="+config.Host)
+		cmd.Env = append(cmd.Env, "DATABRICKS_TOKEN="+config.Token)
+
+		// For the purposes of replacements, use testUser.
+		// Note, users might have overriden /api/2.0/preview/scim/v2/Me but that should not affect the replacement:
+		user = testUser
+	} else {
+		// Use whatever authentication mechanism is configured by the test runner.
+		workspaceClient, err = databricks.NewWorkspaceClient(&databricks.Config{})
+		require.NoError(t, err)
+		pUser, err := workspaceClient.CurrentUser.Me(context.Background())
+		require.NoError(t, err, "Failed to get current user")
+		user = *pUser
 	}
 
+	testdiff.PrepareReplacementsUser(t, &repls, user)
+	testdiff.PrepareReplacementsWorkspaceClient(t, &repls, workspaceClient)
+
+	// Must be added PrepareReplacementsUser, otherwise conflicts with [USERNAME]
+	testdiff.PrepareReplacementsUUID(t, &repls)
+
+	// User replacements come last:
+	repls.Repls = append(repls.Repls, config.Repls...)
+
 	if coverDir != "" {
 		// Creating individual coverage directory for each test, because writing to the same one
 		// results in sporadic failures like this one (only if tests are running in parallel):
@@ -295,15 +321,6 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir)
 	}
 
-	// Each local test should use a new token that will result into a new fake workspace,
-	// so that test don't interfere with each other.
-	if cloudEnv == "" {
-		tokenSuffix := strings.ReplaceAll(uuid.NewString(), "-", "")
-		token := "dbapi" + tokenSuffix
-		cmd.Env = append(cmd.Env, "DATABRICKS_TOKEN="+token)
-		repls.Set(token, "[DATABRICKS_TOKEN]")
-	}
-
 	// Write combined output to a file
 	out, err := os.Create(filepath.Join(tmpDir, "output.txt"))
 	require.NoError(t, err)
@@ -320,7 +337,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 
 		for _, req := range server.Requests {
 			reqJson, err := json.MarshalIndent(req, "", "  ")
-			require.NoError(t, err)
+			require.NoErrorf(t, err, "Failed to indent: %#v", req)
 
 			reqJsonWithRepls := repls.Replace(string(reqJson))
 			_, err = f.WriteString(reqJsonWithRepls + "\n")
diff --git a/acceptance/auth/bundle_and_profile/output.txt b/acceptance/auth/bundle_and_profile/output.txt
index 022b3148d..8d2584622 100644
--- a/acceptance/auth/bundle_and_profile/output.txt
+++ b/acceptance/auth/bundle_and_profile/output.txt
@@ -13,13 +13,13 @@
 
 === Inside the bundle, profile flag not matching bundle host. Badness: should use profile from flag instead and not fail
 >>> errcode [CLI] current-user me -p profile_name
-Error: cannot resolve bundle auth configuration: config host mismatch: profile uses host https://non-existing-subdomain.databricks.com, but CLI configured to use [DATABRICKS_URL]
+Error: cannot resolve bundle auth configuration: config host mismatch: profile uses host https://non-existing-subdomain.databricks.com, but CLI configured to use [DATABRICKS_TARGET]
 
 Exit code: 1
 
 === Inside the bundle, target and not matching profile
 >>> errcode [CLI] current-user me -t dev -p profile_name
-Error: cannot resolve bundle auth configuration: config host mismatch: profile uses host https://non-existing-subdomain.databricks.com, but CLI configured to use [DATABRICKS_URL]
+Error: cannot resolve bundle auth configuration: config host mismatch: profile uses host https://non-existing-subdomain.databricks.com, but CLI configured to use [DATABRICKS_TARGET]
 
 Exit code: 1
 
diff --git a/acceptance/auth/bundle_and_profile/test.toml b/acceptance/auth/bundle_and_profile/test.toml
index b20190ca5..1a611ed95 100644
--- a/acceptance/auth/bundle_and_profile/test.toml
+++ b/acceptance/auth/bundle_and_profile/test.toml
@@ -5,4 +5,8 @@ Badness = "When -p flag is used inside the bundle folder for any CLI commands, C
 # This is a workaround to replace DATABRICKS_URL with DATABRICKS_HOST
 [[Repls]]
 Old='DATABRICKS_HOST'
-New='DATABRICKS_URL'
+New='DATABRICKS_TARGET'
+
+[[Repls]]
+Old='DATABRICKS_URL'
+New='DATABRICKS_TARGET'
diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go
index c8a52f4cd..d3db06003 100644
--- a/acceptance/cmd_server_test.go
+++ b/acceptance/cmd_server_test.go
@@ -14,11 +14,7 @@ import (
 
 func StartCmdServer(t *testing.T) *testserver.Server {
 	server := testserver.New(t)
-
-	// {$} is a wildcard that only matches the end of the URL. We explicitly use
-	// /{$} to disambiguate it from the generic handler for '/' which is used to
-	// identify unhandled API endpoints in the test server.
-	server.Handle("/{$}", func(w *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/", func(_ *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		q := r.URL.Query()
 		args := strings.Split(q.Get("args"), " ")
 
diff --git a/acceptance/selftest/out.hello.txt b/acceptance/selftest/basic/out.hello.txt
similarity index 100%
rename from acceptance/selftest/out.hello.txt
rename to acceptance/selftest/basic/out.hello.txt
diff --git a/acceptance/selftest/output.txt b/acceptance/selftest/basic/output.txt
similarity index 100%
rename from acceptance/selftest/output.txt
rename to acceptance/selftest/basic/output.txt
diff --git a/acceptance/selftest/script b/acceptance/selftest/basic/script
similarity index 100%
rename from acceptance/selftest/script
rename to acceptance/selftest/basic/script
diff --git a/acceptance/selftest/test.toml b/acceptance/selftest/basic/test.toml
similarity index 100%
rename from acceptance/selftest/test.toml
rename to acceptance/selftest/basic/test.toml
diff --git a/acceptance/selftest/server/out.requests.txt b/acceptance/selftest/server/out.requests.txt
new file mode 100644
index 000000000..2cb8708ac
--- /dev/null
+++ b/acceptance/selftest/server/out.requests.txt
@@ -0,0 +1,8 @@
+{
+  "method": "GET",
+  "path": "/api/2.0/preview/scim/v2/Me"
+}
+{
+  "method": "GET",
+  "path": "/custom/endpoint"
+}
diff --git a/acceptance/selftest/server/output.txt b/acceptance/selftest/server/output.txt
new file mode 100644
index 000000000..f9e51caa9
--- /dev/null
+++ b/acceptance/selftest/server/output.txt
@@ -0,0 +1,15 @@
+
+>>> curl -s [DATABRICKS_URL]/api/2.0/preview/scim/v2/Me
+{
+    "id": "[USERID]",
+    "userName": "[USERNAME]"
+}
+>>> curl -sD - [DATABRICKS_URL]/custom/endpoint?query=param
+HTTP/1.1 201 Created
+Content-Type: application/json
+Date: (redacted)
+Content-Length: (redacted)
+
+custom
+---
+response
diff --git a/acceptance/selftest/server/script b/acceptance/selftest/server/script
new file mode 100644
index 000000000..53e2c4b8a
--- /dev/null
+++ b/acceptance/selftest/server/script
@@ -0,0 +1,2 @@
+trace curl -s $DATABRICKS_HOST/api/2.0/preview/scim/v2/Me
+trace curl -sD - $DATABRICKS_HOST/custom/endpoint?query=param
diff --git a/acceptance/selftest/server/test.toml b/acceptance/selftest/server/test.toml
new file mode 100644
index 000000000..2531fb910
--- /dev/null
+++ b/acceptance/selftest/server/test.toml
@@ -0,0 +1,18 @@
+LocalOnly = true
+RecordRequests = true
+
+[[Server]]
+Pattern = "GET /custom/endpoint"
+Response.Body = '''custom
+---
+response
+'''
+Response.StatusCode = 201
+
+[[Repls]]
+Old = 'Date: .*'
+New = 'Date: (redacted)'
+
+[[Repls]]
+Old = 'Content-Length: [0-9]*'
+New = 'Content-Length: (redacted)'
diff --git a/acceptance/server_test.go b/acceptance/server_test.go
index d21ab66e8..11d03c30b 100644
--- a/acceptance/server_test.go
+++ b/acceptance/server_test.go
@@ -8,6 +8,7 @@ import (
 
 	"github.com/databricks/databricks-sdk-go/service/catalog"
 	"github.com/databricks/databricks-sdk-go/service/iam"
+	"github.com/gorilla/mux"
 
 	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
@@ -16,8 +17,13 @@ import (
 	"github.com/databricks/databricks-sdk-go/service/workspace"
 )
 
+var testUser = iam.User{
+	Id:       "1000012345",
+	UserName: "tester@databricks.com",
+}
+
 func AddHandlers(server *testserver.Server) {
-	server.Handle("GET /api/2.0/policies/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/api/2.0/policies/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		return compute.ListPoliciesResponse{
 			Policies: []compute.Policy{
 				{
@@ -32,7 +38,7 @@ func AddHandlers(server *testserver.Server) {
 		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.0/instance-pools/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/api/2.0/instance-pools/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		return compute.ListInstancePools{
 			InstancePools: []compute.InstancePoolAndStats{
 				{
@@ -43,7 +49,7 @@ func AddHandlers(server *testserver.Server) {
 		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.1/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/api/2.1/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		return compute.ListClustersResponse{
 			Clusters: []compute.ClusterDetails{
 				{
@@ -58,20 +64,17 @@ func AddHandlers(server *testserver.Server) {
 		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.0/preview/scim/v2/Me", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		return iam.User{
-			Id:       "1000012345",
-			UserName: "tester@databricks.com",
-		}, http.StatusOK
+	server.Handle("GET", "/api/2.0/preview/scim/v2/Me", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		return testUser, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.0/workspace/get-status", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/api/2.0/workspace/get-status", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		path := r.URL.Query().Get("path")
 
 		return fakeWorkspace.WorkspaceGetStatus(path)
 	})
 
-	server.Handle("POST /api/2.0/workspace/mkdirs", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("POST", "/api/2.0/workspace/mkdirs", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		request := workspace.Mkdirs{}
 		decoder := json.NewDecoder(r.Body)
 
@@ -83,13 +86,13 @@ func AddHandlers(server *testserver.Server) {
 		return fakeWorkspace.WorkspaceMkdirs(request)
 	})
 
-	server.Handle("GET /api/2.0/workspace/export", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/api/2.0/workspace/export", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		path := r.URL.Query().Get("path")
 
 		return fakeWorkspace.WorkspaceExport(path)
 	})
 
-	server.Handle("POST /api/2.0/workspace/delete", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("POST", "/api/2.0/workspace/delete", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		path := r.URL.Query().Get("path")
 		recursiveStr := r.URL.Query().Get("recursive")
 		var recursive bool
@@ -103,8 +106,9 @@ func AddHandlers(server *testserver.Server) {
 		return fakeWorkspace.WorkspaceDelete(path, recursive)
 	})
 
-	server.Handle("POST /api/2.0/workspace-files/import-file/{path}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		path := r.PathValue("path")
+	server.Handle("POST", "/api/2.0/workspace-files/import-file/{path:.*}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		vars := mux.Vars(r)
+		path := vars["path"]
 
 		body := new(bytes.Buffer)
 		_, err := body.ReadFrom(r.Body)
@@ -115,14 +119,15 @@ func AddHandlers(server *testserver.Server) {
 		return fakeWorkspace.WorkspaceFilesImportFile(path, body.Bytes())
 	})
 
-	server.Handle("GET /api/2.1/unity-catalog/current-metastore-assignment", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/api/2.1/unity-catalog/current-metastore-assignment", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		return catalog.MetastoreAssignment{
 			DefaultCatalogName: "main",
 		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.0/permissions/directories/{objectId}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		objectId := r.PathValue("objectId")
+	server.Handle("GET", "/api/2.0/permissions/directories/{objectId}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		vars := mux.Vars(r)
+		objectId := vars["objectId"]
 
 		return workspace.WorkspaceObjectPermissions{
 			ObjectId:   objectId,
@@ -140,7 +145,7 @@ func AddHandlers(server *testserver.Server) {
 		}, http.StatusOK
 	})
 
-	server.Handle("POST /api/2.1/jobs/create", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("POST", "/api/2.1/jobs/create", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		request := jobs.CreateJob{}
 		decoder := json.NewDecoder(r.Body)
 
@@ -152,13 +157,13 @@ func AddHandlers(server *testserver.Server) {
 		return fakeWorkspace.JobsCreate(request)
 	})
 
-	server.Handle("GET /api/2.1/jobs/get", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/api/2.1/jobs/get", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		jobId := r.URL.Query().Get("job_id")
 
 		return fakeWorkspace.JobsGet(jobId)
 	})
 
-	server.Handle("GET /api/2.1/jobs/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/api/2.1/jobs/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		return fakeWorkspace.JobsList()
 	})
 }
diff --git a/go.mod b/go.mod
index c8b209edd..2e2505361 100644
--- a/go.mod
+++ b/go.mod
@@ -12,6 +12,7 @@ require (
 	github.com/databricks/databricks-sdk-go v0.57.0 // Apache 2.0
 	github.com/fatih/color v1.18.0 // MIT
 	github.com/google/uuid v1.6.0 // BSD-3-Clause
+	github.com/gorilla/mux v1.8.1 // BSD 3-Clause
 	github.com/hashicorp/go-version v1.7.0 // MPL 2.0
 	github.com/hashicorp/hc-install v0.9.1 // MPL 2.0
 	github.com/hashicorp/terraform-exec v0.22.0 // MPL 2.0
diff --git a/go.sum b/go.sum
index 0369fc2d9..fbf942148 100644
--- a/go.sum
+++ b/go.sum
@@ -97,6 +97,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF
 github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
 github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg=
 github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
diff --git a/libs/testserver/server.go b/libs/testserver/server.go
index 577ef082c..cf4d5aca2 100644
--- a/libs/testserver/server.go
+++ b/libs/testserver/server.go
@@ -9,6 +9,8 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/gorilla/mux"
+
 	"github.com/stretchr/testify/assert"
 
 	"github.com/databricks/cli/internal/testutil"
@@ -17,7 +19,7 @@ import (
 
 type Server struct {
 	*httptest.Server
-	Mux *http.ServeMux
+	Router *mux.Router
 
 	t testutil.TestingT
 
@@ -39,22 +41,20 @@ type Request struct {
 }
 
 func New(t testutil.TestingT) *Server {
-	mux := http.NewServeMux()
-	server := httptest.NewServer(mux)
+	router := mux.NewRouter()
+	server := httptest.NewServer(router)
 	t.Cleanup(server.Close)
 
 	s := &Server{
 		Server:         server,
-		Mux:            mux,
+		Router:         router,
 		t:              t,
 		mu:             &sync.Mutex{},
 		fakeWorkspaces: map[string]*FakeWorkspace{},
 	}
 
-	// The server resolves conflicting handlers by using the one with higher
-	// specificity. This handler is the least specific, so it will be used as a
-	// fallback when no other handlers match.
-	s.Handle("/", func(fakeWorkspace *FakeWorkspace, r *http.Request) (any, int) {
+	// Set up the not found handler as fallback
+	router.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		pattern := r.Method + " " + r.URL.Path
 
 		t.Errorf(`
@@ -75,9 +75,22 @@ Response.StatusCode = <response status-code here>
 
 `, pattern, pattern)
 
-		return apierr.APIError{
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusNotImplemented)
+
+		resp := apierr.APIError{
 			Message: "No stub found for pattern: " + pattern,
-		}, http.StatusNotImplemented
+		}
+
+		respBytes, err := json.Marshal(resp)
+		if err != nil {
+			t.Errorf("JSON encoding error: %s", err)
+			respBytes = []byte("{\"message\": \"JSON encoding error\"}")
+		}
+
+		if _, err := w.Write(respBytes); err != nil {
+			t.Errorf("Response write error: %s", err)
+		}
 	})
 
 	return s
@@ -85,8 +98,8 @@ Response.StatusCode = <response status-code here>
 
 type HandlerFunc func(fakeWorkspace *FakeWorkspace, req *http.Request) (resp any, statusCode int)
 
-func (s *Server) Handle(pattern string, handler HandlerFunc) {
-	s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) {
+func (s *Server) Handle(method, path string, handler HandlerFunc) {
+	s.Router.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
 		// For simplicity we process requests sequentially. It's fast enough because
 		// we don't do any IO except reading and writing request/response bodies.
 		s.mu.Lock()
@@ -156,7 +169,7 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
-	})
+	}).Methods(method)
 }
 
 func getToken(r *http.Request) string {

From 24ac8d8d595df531a2aa7b9faa72f8be7132f7b1 Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Tue, 11 Feb 2025 21:20:03 +0530
Subject: [PATCH 235/247] Add acceptance tests for auth resolution (#2285)

## Changes

This PR adds acceptance tests for native Databricks auth methods: basic,
oauth, and pat.

In the future we could compare this with auth credentials used by
downstream tools like TF or the telemetry process to ensure consistent
auth credentials are picked up and used.

Note:
We do not add acceptance tests for other auth methods like Azure because
they communicate with external endpoints. To test them locally, we would
need to set up a reverse proxy server, which is out of scope for this
change.

## Tests
N/A
---
 .../auth/credentials/basic/out.requests.txt   | 12 +++++++
 acceptance/auth/credentials/basic/output.txt  |  4 +++
 acceptance/auth/credentials/basic/script      |  8 +++++
 acceptance/auth/credentials/basic/test.toml   |  4 +++
 .../auth/credentials/oauth/out.requests.txt   | 34 +++++++++++++++++++
 acceptance/auth/credentials/oauth/output.txt  |  4 +++
 acceptance/auth/credentials/oauth/script      |  8 +++++
 acceptance/auth/credentials/oauth/test.toml   |  5 +++
 .../auth/credentials/pat/out.requests.txt     | 12 +++++++
 acceptance/auth/credentials/pat/output.txt    |  4 +++
 acceptance/auth/credentials/pat/script        |  3 ++
 acceptance/auth/credentials/test.toml         | 20 +++++++++++
 acceptance/server_test.go                     | 16 +++++++++
 13 files changed, 134 insertions(+)
 create mode 100644 acceptance/auth/credentials/basic/out.requests.txt
 create mode 100644 acceptance/auth/credentials/basic/output.txt
 create mode 100644 acceptance/auth/credentials/basic/script
 create mode 100644 acceptance/auth/credentials/basic/test.toml
 create mode 100644 acceptance/auth/credentials/oauth/out.requests.txt
 create mode 100644 acceptance/auth/credentials/oauth/output.txt
 create mode 100644 acceptance/auth/credentials/oauth/script
 create mode 100644 acceptance/auth/credentials/oauth/test.toml
 create mode 100644 acceptance/auth/credentials/pat/out.requests.txt
 create mode 100644 acceptance/auth/credentials/pat/output.txt
 create mode 100644 acceptance/auth/credentials/pat/script
 create mode 100644 acceptance/auth/credentials/test.toml

diff --git a/acceptance/auth/credentials/basic/out.requests.txt b/acceptance/auth/credentials/basic/out.requests.txt
new file mode 100644
index 000000000..b549c7423
--- /dev/null
+++ b/acceptance/auth/credentials/basic/out.requests.txt
@@ -0,0 +1,12 @@
+{
+  "headers": {
+    "Authorization": [
+      "Basic [ENCODED_AUTH]"
+    ],
+    "User-Agent": [
+      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/current-user_me cmd-exec-id/[UUID] auth/basic"
+    ]
+  },
+  "method": "GET",
+  "path": "/api/2.0/preview/scim/v2/Me"
+}
diff --git a/acceptance/auth/credentials/basic/output.txt b/acceptance/auth/credentials/basic/output.txt
new file mode 100644
index 000000000..c5747c9e4
--- /dev/null
+++ b/acceptance/auth/credentials/basic/output.txt
@@ -0,0 +1,4 @@
+{
+  "id":"[USERID]",
+  "userName":"[USERNAME]"
+}
diff --git a/acceptance/auth/credentials/basic/script b/acceptance/auth/credentials/basic/script
new file mode 100644
index 000000000..aae249083
--- /dev/null
+++ b/acceptance/auth/credentials/basic/script
@@ -0,0 +1,8 @@
+# Unset the token which is configured by default
+# in acceptance tests
+export DATABRICKS_TOKEN=""
+
+export DATABRICKS_USERNAME=username
+export DATABRICKS_PASSWORD=password
+
+$CLI current-user me
diff --git a/acceptance/auth/credentials/basic/test.toml b/acceptance/auth/credentials/basic/test.toml
new file mode 100644
index 000000000..4998d81d7
--- /dev/null
+++ b/acceptance/auth/credentials/basic/test.toml
@@ -0,0 +1,4 @@
+# "username:password" in base64 is dXNlcm5hbWU6cGFzc3dvcmQ=, expect to see this in Authorization header
+[[Repls]]
+Old = "dXNlcm5hbWU6cGFzc3dvcmQ="
+New = "[ENCODED_AUTH]"
diff --git a/acceptance/auth/credentials/oauth/out.requests.txt b/acceptance/auth/credentials/oauth/out.requests.txt
new file mode 100644
index 000000000..525e148d8
--- /dev/null
+++ b/acceptance/auth/credentials/oauth/out.requests.txt
@@ -0,0 +1,34 @@
+{
+  "headers": {
+    "User-Agent": [
+      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]"
+    ]
+  },
+  "method": "GET",
+  "path": "/oidc/.well-known/oauth-authorization-server"
+}
+{
+  "headers": {
+    "Authorization": [
+      "Basic [ENCODED_AUTH]"
+    ],
+    "User-Agent": [
+      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]"
+    ]
+  },
+  "method": "POST",
+  "path": "/oidc/v1/token",
+  "raw_body": "grant_type=client_credentials\u0026scope=all-apis"
+}
+{
+  "headers": {
+    "Authorization": [
+      "Bearer oauth-token"
+    ],
+    "User-Agent": [
+      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/current-user_me cmd-exec-id/[UUID] auth/oauth-m2m"
+    ]
+  },
+  "method": "GET",
+  "path": "/api/2.0/preview/scim/v2/Me"
+}
diff --git a/acceptance/auth/credentials/oauth/output.txt b/acceptance/auth/credentials/oauth/output.txt
new file mode 100644
index 000000000..c5747c9e4
--- /dev/null
+++ b/acceptance/auth/credentials/oauth/output.txt
@@ -0,0 +1,4 @@
+{
+  "id":"[USERID]",
+  "userName":"[USERNAME]"
+}
diff --git a/acceptance/auth/credentials/oauth/script b/acceptance/auth/credentials/oauth/script
new file mode 100644
index 000000000..e4519e41b
--- /dev/null
+++ b/acceptance/auth/credentials/oauth/script
@@ -0,0 +1,8 @@
+# Unset the token which is configured by default
+# in acceptance tests
+export DATABRICKS_TOKEN=""
+
+export DATABRICKS_CLIENT_ID=client_id
+export DATABRICKS_CLIENT_SECRET=client_secret
+
+$CLI current-user me
diff --git a/acceptance/auth/credentials/oauth/test.toml b/acceptance/auth/credentials/oauth/test.toml
new file mode 100644
index 000000000..2adade96a
--- /dev/null
+++ b/acceptance/auth/credentials/oauth/test.toml
@@ -0,0 +1,5 @@
+# "client_id:client_secret" in base64 is Y2xpZW50X2lkOmNsaWVudF9zZWNyZXQ=, expect to
+# see this in Authorization header
+[[Repls]]
+Old = "Y2xpZW50X2lkOmNsaWVudF9zZWNyZXQ="
+New = "[ENCODED_AUTH]"
diff --git a/acceptance/auth/credentials/pat/out.requests.txt b/acceptance/auth/credentials/pat/out.requests.txt
new file mode 100644
index 000000000..73c448c2f
--- /dev/null
+++ b/acceptance/auth/credentials/pat/out.requests.txt
@@ -0,0 +1,12 @@
+{
+  "headers": {
+    "Authorization": [
+      "Bearer dapi1234"
+    ],
+    "User-Agent": [
+      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/current-user_me cmd-exec-id/[UUID] auth/pat"
+    ]
+  },
+  "method": "GET",
+  "path": "/api/2.0/preview/scim/v2/Me"
+}
diff --git a/acceptance/auth/credentials/pat/output.txt b/acceptance/auth/credentials/pat/output.txt
new file mode 100644
index 000000000..c5747c9e4
--- /dev/null
+++ b/acceptance/auth/credentials/pat/output.txt
@@ -0,0 +1,4 @@
+{
+  "id":"[USERID]",
+  "userName":"[USERNAME]"
+}
diff --git a/acceptance/auth/credentials/pat/script b/acceptance/auth/credentials/pat/script
new file mode 100644
index 000000000..ccf1098e7
--- /dev/null
+++ b/acceptance/auth/credentials/pat/script
@@ -0,0 +1,3 @@
+export DATABRICKS_TOKEN=dapi1234
+
+$CLI current-user me
diff --git a/acceptance/auth/credentials/test.toml b/acceptance/auth/credentials/test.toml
new file mode 100644
index 000000000..89438f43a
--- /dev/null
+++ b/acceptance/auth/credentials/test.toml
@@ -0,0 +1,20 @@
+LocalOnly = true
+
+RecordRequests = true
+IncludeRequestHeaders = ["Authorization", "User-Agent"]
+
+[[Repls]]
+Old = '(linux|darwin|windows)'
+New = '[OS]'
+
+[[Repls]]
+Old = " upstream/[A-Za-z0-9.-]+"
+New = ""
+
+[[Repls]]
+Old = " upstream-version/[A-Za-z0-9.-]+"
+New = ""
+
+[[Repls]]
+Old = " cicd/[A-Za-z0-9.-]+"
+New = ""
diff --git a/acceptance/server_test.go b/acceptance/server_test.go
index 11d03c30b..fd8006b8f 100644
--- a/acceptance/server_test.go
+++ b/acceptance/server_test.go
@@ -166,6 +166,22 @@ func AddHandlers(server *testserver.Server) {
 	server.Handle("GET", "/api/2.1/jobs/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		return fakeWorkspace.JobsList()
 	})
+
+	server.Handle("GET", "/oidc/.well-known/oauth-authorization-server", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		return map[string]string{
+			"authorization_endpoint": server.URL + "oidc/v1/authorize",
+			"token_endpoint":         server.URL + "/oidc/v1/token",
+		}, http.StatusOK
+	})
+
+	server.Handle("POST", "/oidc/v1/token", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		return map[string]string{
+			"access_token": "oauth-token",
+			"expires_in":   "3600",
+			"scope":        "all-apis",
+			"token_type":   "Bearer",
+		}, http.StatusOK
+	})
 }
 
 func internalError(err error) (any, int) {

From bfde3585b9df9eb5f38a4bed63d2f85426ad1119 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 12 Feb 2025 12:45:01 +0100
Subject: [PATCH 236/247] acc: Fix priority of stubs in test.toml (#2339)

## Changes
Reverse the order of stubs to match expectation (leaf configuration
takes precedence over parent configuration).
Follow up to #2334 .

## Tests
acceptance/selftest/server is extended with duplicate handler
---
 acceptance/acceptance_test.go        | 4 ++++
 acceptance/selftest/server/test.toml | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 117172f60..e61166c31 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -259,6 +259,10 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 			server.RecordRequests = config.RecordRequests
 			server.IncludeRequestHeaders = config.IncludeRequestHeaders
 
+			// We want later stubs takes precedence, because then leaf configs take precedence over parent directory configs
+			// In gorilla/mux earlier handlers take precedence, so we need to reverse the order
+			slices.Reverse(config.Server)
+
 			for _, stub := range config.Server {
 				require.NotEmpty(t, stub.Pattern)
 				items := strings.Split(stub.Pattern, " ")
diff --git a/acceptance/selftest/server/test.toml b/acceptance/selftest/server/test.toml
index 2531fb910..fca41bf02 100644
--- a/acceptance/selftest/server/test.toml
+++ b/acceptance/selftest/server/test.toml
@@ -1,6 +1,10 @@
 LocalOnly = true
 RecordRequests = true
 
+[[Server]]
+Pattern = "GET /custom/endpoint"
+Response.Body = '''should not see this response, latter response takes precedence'''
+
 [[Server]]
 Pattern = "GET /custom/endpoint"
 Response.Body = '''custom

From 4034766c939620c062a0dbad2207eaad6821c54b Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 12 Feb 2025 14:00:57 +0100
Subject: [PATCH 237/247] acc: Simplify writing handlers; support headers in
 responses (#2338)

## Changes

Handlers now receive testserver.Request and return any which could be
- string or []byte (returns it as is but sets content-type to json or
plain text depending on content)
- struct (encodes it as json and sets content-type to json)
- testserver.Response (full control over status and headers)

Note if testserver.Response is returned from the handler, it's Body
attribute can still be an object. In that case, it'll be serialized and
appropriate content-type header will be added.

The config is now using the same testserver.Response struct, the same
logic applies both configured responses and responses returned from
handlers.

As a result, one can set headers both in Golang handlers and in
test.toml.

This also fixes a bug with RecordRequest not seeing the body if it was
already consumed by the handler.

## Tests
- Existing rests.
- acceptance/selftest/server is extended to set response header.
---
 acceptance/acceptance_test.go                 |   9 +-
 .../bundle/debug/out.stderr.parallel.txt      |   6 +-
 acceptance/bundle/debug/out.stderr.txt        |   3 +-
 acceptance/cmd_server_test.go                 |   8 +-
 acceptance/config_test.go                     |   6 +-
 acceptance/selftest/server/out.requests.txt   |   4 +
 acceptance/selftest/server/output.txt         |   8 +-
 acceptance/selftest/server/script             |   2 +
 acceptance/selftest/server/test.toml          |   2 +
 acceptance/server_test.go                     | 132 ++++-----
 libs/testserver/fake_workspace.go             |  98 +++----
 libs/testserver/server.go                     | 251 ++++++++++++++----
 12 files changed, 336 insertions(+), 193 deletions(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index e61166c31..85c345032 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -7,7 +7,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -267,12 +266,8 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 				require.NotEmpty(t, stub.Pattern)
 				items := strings.Split(stub.Pattern, " ")
 				require.Len(t, items, 2)
-				server.Handle(items[0], items[1], func(fakeWorkspace *testserver.FakeWorkspace, req *http.Request) (any, int) {
-					statusCode := http.StatusOK
-					if stub.Response.StatusCode != 0 {
-						statusCode = stub.Response.StatusCode
-					}
-					return stub.Response.Body, statusCode
+				server.Handle(items[0], items[1], func(req testserver.Request) any {
+					return stub.Response
 				})
 			}
 
diff --git a/acceptance/bundle/debug/out.stderr.parallel.txt b/acceptance/bundle/debug/out.stderr.parallel.txt
index 7dd770068..13c81c511 100644
--- a/acceptance/bundle/debug/out.stderr.parallel.txt
+++ b/acceptance/bundle/debug/out.stderr.parallel.txt
@@ -9,7 +9,7 @@
 10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:folder_permissions
 10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:validate_sync_patterns
 10:07:59 Debug: Path /Workspace/Users/[USERNAME]/.bundle/debug/default/files has type directory (ID: 0) pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync
-10:07:59 Debug: non-retriable error:  pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
-< {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
-< {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
+10:07:59 Debug: non-retriable error: Workspace path not found pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
+< HTTP/0.0 000 OK pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
+< } pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
 < } pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
diff --git a/acceptance/bundle/debug/out.stderr.txt b/acceptance/bundle/debug/out.stderr.txt
index 9cac8bb2b..e5867e008 100644
--- a/acceptance/bundle/debug/out.stderr.txt
+++ b/acceptance/bundle/debug/out.stderr.txt
@@ -79,11 +79,12 @@
 10:07:59 Debug: Apply pid=12345 mutator=validate
 10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files
 < HTTP/1.1 404 Not Found
+< {
+<   "message": "Workspace path not found"
 10:07:59 Debug: POST /api/2.0/workspace/mkdirs
 > {
 >   "path": "/Workspace/Users/[USERNAME]/.bundle/debug/default/files"
 > }
-< HTTP/1.1 200 OK
 10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files
 < HTTP/1.1 200 OK
 < {
diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go
index d3db06003..dc48a85d7 100644
--- a/acceptance/cmd_server_test.go
+++ b/acceptance/cmd_server_test.go
@@ -1,8 +1,8 @@
 package acceptance_test
 
 import (
+	"context"
 	"encoding/json"
-	"net/http"
 	"os"
 	"strings"
 	"testing"
@@ -14,7 +14,7 @@ import (
 
 func StartCmdServer(t *testing.T) *testserver.Server {
 	server := testserver.New(t)
-	server.Handle("GET", "/", func(_ *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/", func(r testserver.Request) any {
 		q := r.URL.Query()
 		args := strings.Split(q.Get("args"), " ")
 
@@ -27,7 +27,7 @@ func StartCmdServer(t *testing.T) *testserver.Server {
 
 		defer Chdir(t, q.Get("cwd"))()
 
-		c := testcli.NewRunner(t, r.Context(), args...)
+		c := testcli.NewRunner(t, context.Background(), args...)
 		c.Verbose = false
 		stdout, stderr, err := c.Run()
 		result := map[string]any{
@@ -39,7 +39,7 @@ func StartCmdServer(t *testing.T) *testserver.Server {
 			exitcode = 1
 		}
 		result["exitcode"] = exitcode
-		return result, http.StatusOK
+		return result
 	})
 	return server
 }
diff --git a/acceptance/config_test.go b/acceptance/config_test.go
index 920e713a1..ec0d1baee 100644
--- a/acceptance/config_test.go
+++ b/acceptance/config_test.go
@@ -10,6 +10,7 @@ import (
 	"dario.cat/mergo"
 	"github.com/BurntSushi/toml"
 	"github.com/databricks/cli/libs/testdiff"
+	"github.com/databricks/cli/libs/testserver"
 	"github.com/stretchr/testify/require"
 )
 
@@ -56,10 +57,7 @@ type ServerStub struct {
 	Pattern string
 
 	// The response body to return.
-	Response struct {
-		Body       string
-		StatusCode int
-	}
+	Response testserver.Response
 }
 
 // FindConfigs finds all the config relevant for this test,
diff --git a/acceptance/selftest/server/out.requests.txt b/acceptance/selftest/server/out.requests.txt
index 2cb8708ac..34f4c4899 100644
--- a/acceptance/selftest/server/out.requests.txt
+++ b/acceptance/selftest/server/out.requests.txt
@@ -6,3 +6,7 @@
   "method": "GET",
   "path": "/custom/endpoint"
 }
+{
+  "method": "GET",
+  "path": "/api/2.0/workspace/get-status"
+}
diff --git a/acceptance/selftest/server/output.txt b/acceptance/selftest/server/output.txt
index f9e51caa9..7147f9c9b 100644
--- a/acceptance/selftest/server/output.txt
+++ b/acceptance/selftest/server/output.txt
@@ -6,10 +6,16 @@
 }
 >>> curl -sD - [DATABRICKS_URL]/custom/endpoint?query=param
 HTTP/1.1 201 Created
-Content-Type: application/json
+X-Custom-Header: hello
 Date: (redacted)
 Content-Length: (redacted)
+Content-Type: text/plain; charset=utf-8
 
 custom
 ---
 response
+
+>>> errcode [CLI] workspace get-status /a/b/c
+Error: Workspace path not found
+
+Exit code: 1
diff --git a/acceptance/selftest/server/script b/acceptance/selftest/server/script
index 53e2c4b8a..810ea64b6 100644
--- a/acceptance/selftest/server/script
+++ b/acceptance/selftest/server/script
@@ -1,2 +1,4 @@
 trace curl -s $DATABRICKS_HOST/api/2.0/preview/scim/v2/Me
 trace curl -sD - $DATABRICKS_HOST/custom/endpoint?query=param
+
+trace errcode $CLI workspace get-status /a/b/c
diff --git a/acceptance/selftest/server/test.toml b/acceptance/selftest/server/test.toml
index fca41bf02..43ad1e85b 100644
--- a/acceptance/selftest/server/test.toml
+++ b/acceptance/selftest/server/test.toml
@@ -12,6 +12,8 @@ Response.Body = '''custom
 response
 '''
 Response.StatusCode = 201
+[Server.Response.Headers]
+"X-Custom-Header" = ["hello"]
 
 [[Repls]]
 Old = 'Date: .*'
diff --git a/acceptance/server_test.go b/acceptance/server_test.go
index fd8006b8f..f73872e0b 100644
--- a/acceptance/server_test.go
+++ b/acceptance/server_test.go
@@ -1,14 +1,12 @@
 package acceptance_test
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"net/http"
 
 	"github.com/databricks/databricks-sdk-go/service/catalog"
 	"github.com/databricks/databricks-sdk-go/service/iam"
-	"github.com/gorilla/mux"
 
 	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
@@ -23,7 +21,7 @@ var testUser = iam.User{
 }
 
 func AddHandlers(server *testserver.Server) {
-	server.Handle("GET", "/api/2.0/policies/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/api/2.0/policies/clusters/list", func(req testserver.Request) any {
 		return compute.ListPoliciesResponse{
 			Policies: []compute.Policy{
 				{
@@ -35,10 +33,10 @@ func AddHandlers(server *testserver.Server) {
 					Name:     "some-test-cluster-policy",
 				},
 			},
-		}, http.StatusOK
+		}
 	})
 
-	server.Handle("GET", "/api/2.0/instance-pools/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/api/2.0/instance-pools/list", func(req testserver.Request) any {
 		return compute.ListInstancePools{
 			InstancePools: []compute.InstancePoolAndStats{
 				{
@@ -46,10 +44,10 @@ func AddHandlers(server *testserver.Server) {
 					InstancePoolId:   "1234",
 				},
 			},
-		}, http.StatusOK
+		}
 	})
 
-	server.Handle("GET", "/api/2.1/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/api/2.1/clusters/list", func(req testserver.Request) any {
 		return compute.ListClustersResponse{
 			Clusters: []compute.ClusterDetails{
 				{
@@ -61,74 +59,57 @@ func AddHandlers(server *testserver.Server) {
 					ClusterId:   "9876",
 				},
 			},
-		}, http.StatusOK
+		}
 	})
 
-	server.Handle("GET", "/api/2.0/preview/scim/v2/Me", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		return testUser, http.StatusOK
+	server.Handle("GET", "/api/2.0/preview/scim/v2/Me", func(req testserver.Request) any {
+		return testUser
 	})
 
-	server.Handle("GET", "/api/2.0/workspace/get-status", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		path := r.URL.Query().Get("path")
-
-		return fakeWorkspace.WorkspaceGetStatus(path)
+	server.Handle("GET", "/api/2.0/workspace/get-status", func(req testserver.Request) any {
+		path := req.URL.Query().Get("path")
+		return req.Workspace.WorkspaceGetStatus(path)
 	})
 
-	server.Handle("POST", "/api/2.0/workspace/mkdirs", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		request := workspace.Mkdirs{}
-		decoder := json.NewDecoder(r.Body)
-
-		err := decoder.Decode(&request)
-		if err != nil {
-			return internalError(err)
+	server.Handle("POST", "/api/2.0/workspace/mkdirs", func(req testserver.Request) any {
+		var request workspace.Mkdirs
+		if err := json.Unmarshal(req.Body, &request); err != nil {
+			return testserver.Response{
+				Body:       fmt.Sprintf("internal error: %s", err),
+				StatusCode: http.StatusInternalServerError,
+			}
 		}
 
-		return fakeWorkspace.WorkspaceMkdirs(request)
+		req.Workspace.WorkspaceMkdirs(request)
+		return ""
 	})
 
-	server.Handle("GET", "/api/2.0/workspace/export", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		path := r.URL.Query().Get("path")
-
-		return fakeWorkspace.WorkspaceExport(path)
+	server.Handle("GET", "/api/2.0/workspace/export", func(req testserver.Request) any {
+		path := req.URL.Query().Get("path")
+		return req.Workspace.WorkspaceExport(path)
 	})
 
-	server.Handle("POST", "/api/2.0/workspace/delete", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		path := r.URL.Query().Get("path")
-		recursiveStr := r.URL.Query().Get("recursive")
-		var recursive bool
-
-		if recursiveStr == "true" {
-			recursive = true
-		} else {
-			recursive = false
-		}
-
-		return fakeWorkspace.WorkspaceDelete(path, recursive)
+	server.Handle("POST", "/api/2.0/workspace/delete", func(req testserver.Request) any {
+		path := req.URL.Query().Get("path")
+		recursive := req.URL.Query().Get("recursive") == "true"
+		req.Workspace.WorkspaceDelete(path, recursive)
+		return ""
 	})
 
-	server.Handle("POST", "/api/2.0/workspace-files/import-file/{path:.*}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		vars := mux.Vars(r)
-		path := vars["path"]
-
-		body := new(bytes.Buffer)
-		_, err := body.ReadFrom(r.Body)
-		if err != nil {
-			return internalError(err)
-		}
-
-		return fakeWorkspace.WorkspaceFilesImportFile(path, body.Bytes())
+	server.Handle("POST", "/api/2.0/workspace-files/import-file/{path:.*}", func(req testserver.Request) any {
+		path := req.Vars["path"]
+		req.Workspace.WorkspaceFilesImportFile(path, req.Body)
+		return ""
 	})
 
-	server.Handle("GET", "/api/2.1/unity-catalog/current-metastore-assignment", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/api/2.1/unity-catalog/current-metastore-assignment", func(req testserver.Request) any {
 		return catalog.MetastoreAssignment{
 			DefaultCatalogName: "main",
-		}, http.StatusOK
+		}
 	})
 
-	server.Handle("GET", "/api/2.0/permissions/directories/{objectId}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		vars := mux.Vars(r)
-		objectId := vars["objectId"]
-
+	server.Handle("GET", "/api/2.0/permissions/directories/{objectId}", func(req testserver.Request) any {
+		objectId := req.Vars["objectId"]
 		return workspace.WorkspaceObjectPermissions{
 			ObjectId:   objectId,
 			ObjectType: "DIRECTORY",
@@ -142,48 +123,43 @@ func AddHandlers(server *testserver.Server) {
 					},
 				},
 			},
-		}, http.StatusOK
+		}
 	})
 
-	server.Handle("POST", "/api/2.1/jobs/create", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		request := jobs.CreateJob{}
-		decoder := json.NewDecoder(r.Body)
-
-		err := decoder.Decode(&request)
-		if err != nil {
-			return internalError(err)
+	server.Handle("POST", "/api/2.1/jobs/create", func(req testserver.Request) any {
+		var request jobs.CreateJob
+		if err := json.Unmarshal(req.Body, &request); err != nil {
+			return testserver.Response{
+				Body:       fmt.Sprintf("internal error: %s", err),
+				StatusCode: 500,
+			}
 		}
 
-		return fakeWorkspace.JobsCreate(request)
+		return req.Workspace.JobsCreate(request)
 	})
 
-	server.Handle("GET", "/api/2.1/jobs/get", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		jobId := r.URL.Query().Get("job_id")
-
-		return fakeWorkspace.JobsGet(jobId)
+	server.Handle("GET", "/api/2.1/jobs/get", func(req testserver.Request) any {
+		jobId := req.URL.Query().Get("job_id")
+		return req.Workspace.JobsGet(jobId)
 	})
 
-	server.Handle("GET", "/api/2.1/jobs/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		return fakeWorkspace.JobsList()
+	server.Handle("GET", "/api/2.1/jobs/list", func(req testserver.Request) any {
+		return req.Workspace.JobsList()
 	})
 
-	server.Handle("GET", "/oidc/.well-known/oauth-authorization-server", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/oidc/.well-known/oauth-authorization-server", func(_ testserver.Request) any {
 		return map[string]string{
 			"authorization_endpoint": server.URL + "oidc/v1/authorize",
 			"token_endpoint":         server.URL + "/oidc/v1/token",
-		}, http.StatusOK
+		}
 	})
 
-	server.Handle("POST", "/oidc/v1/token", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("POST", "/oidc/v1/token", func(_ testserver.Request) any {
 		return map[string]string{
 			"access_token": "oauth-token",
 			"expires_in":   "3600",
 			"scope":        "all-apis",
 			"token_type":   "Bearer",
-		}, http.StatusOK
+		}
 	})
 }
-
-func internalError(err error) (any, int) {
-	return fmt.Errorf("internal error: %w", err), http.StatusInternalServerError
-}
diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go
index c3e4f9a71..4e943f828 100644
--- a/libs/testserver/fake_workspace.go
+++ b/libs/testserver/fake_workspace.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"net/http"
 	"sort"
 	"strconv"
 	"strings"
@@ -33,40 +32,39 @@ func NewFakeWorkspace() *FakeWorkspace {
 	}
 }
 
-func (s *FakeWorkspace) WorkspaceGetStatus(path string) (workspace.ObjectInfo, int) {
+func (s *FakeWorkspace) WorkspaceGetStatus(path string) Response {
 	if s.directories[path] {
-		return workspace.ObjectInfo{
-			ObjectType: "DIRECTORY",
-			Path:       path,
-		}, http.StatusOK
+		return Response{
+			Body: &workspace.ObjectInfo{
+				ObjectType: "DIRECTORY",
+				Path:       path,
+			},
+		}
 	} else if _, ok := s.files[path]; ok {
-		return workspace.ObjectInfo{
-			ObjectType: "FILE",
-			Path:       path,
-			Language:   "SCALA",
-		}, http.StatusOK
+		return Response{
+			Body: &workspace.ObjectInfo{
+				ObjectType: "FILE",
+				Path:       path,
+				Language:   "SCALA",
+			},
+		}
 	} else {
-		return workspace.ObjectInfo{}, http.StatusNotFound
+		return Response{
+			StatusCode: 404,
+			Body:       map[string]string{"message": "Workspace path not found"},
+		}
 	}
 }
 
-func (s *FakeWorkspace) WorkspaceMkdirs(request workspace.Mkdirs) (string, int) {
+func (s *FakeWorkspace) WorkspaceMkdirs(request workspace.Mkdirs) {
 	s.directories[request.Path] = true
-
-	return "{}", http.StatusOK
 }
 
-func (s *FakeWorkspace) WorkspaceExport(path string) ([]byte, int) {
-	file := s.files[path]
-
-	if file == nil {
-		return nil, http.StatusNotFound
-	}
-
-	return file, http.StatusOK
+func (s *FakeWorkspace) WorkspaceExport(path string) []byte {
+	return s.files[path]
 }
 
-func (s *FakeWorkspace) WorkspaceDelete(path string, recursive bool) (string, int) {
+func (s *FakeWorkspace) WorkspaceDelete(path string, recursive bool) {
 	if !recursive {
 		s.files[path] = nil
 	} else {
@@ -76,28 +74,26 @@ func (s *FakeWorkspace) WorkspaceDelete(path string, recursive bool) (string, in
 			}
 		}
 	}
-
-	return "{}", http.StatusOK
 }
 
-func (s *FakeWorkspace) WorkspaceFilesImportFile(path string, body []byte) (any, int) {
+func (s *FakeWorkspace) WorkspaceFilesImportFile(path string, body []byte) {
 	if !strings.HasPrefix(path, "/") {
 		path = "/" + path
 	}
-
 	s.files[path] = body
-
-	return "{}", http.StatusOK
 }
 
-func (s *FakeWorkspace) JobsCreate(request jobs.CreateJob) (any, int) {
+func (s *FakeWorkspace) JobsCreate(request jobs.CreateJob) Response {
 	jobId := s.nextJobId
 	s.nextJobId++
 
 	jobSettings := jobs.JobSettings{}
 	err := jsonConvert(request, &jobSettings)
 	if err != nil {
-		return internalError(err)
+		return Response{
+			StatusCode: 400,
+			Body:       fmt.Sprintf("Cannot convert request to jobSettings: %s", err),
+		}
 	}
 
 	s.jobs[jobId] = jobs.Job{
@@ -105,32 +101,44 @@ func (s *FakeWorkspace) JobsCreate(request jobs.CreateJob) (any, int) {
 		Settings: &jobSettings,
 	}
 
-	return jobs.CreateResponse{JobId: jobId}, http.StatusOK
+	return Response{
+		Body: jobs.CreateResponse{JobId: jobId},
+	}
 }
 
-func (s *FakeWorkspace) JobsGet(jobId string) (any, int) {
+func (s *FakeWorkspace) JobsGet(jobId string) Response {
 	id := jobId
 
 	jobIdInt, err := strconv.ParseInt(id, 10, 64)
 	if err != nil {
-		return internalError(fmt.Errorf("failed to parse job id: %s", err))
+		return Response{
+			StatusCode: 400,
+			Body:       fmt.Sprintf("Failed to parse job id: %s: %v", err, id),
+		}
 	}
 
 	job, ok := s.jobs[jobIdInt]
 	if !ok {
-		return jobs.Job{}, http.StatusNotFound
+		return Response{
+			StatusCode: 404,
+		}
 	}
 
-	return job, http.StatusOK
+	return Response{
+		Body: job,
+	}
 }
 
-func (s *FakeWorkspace) JobsList() (any, int) {
+func (s *FakeWorkspace) JobsList() Response {
 	list := make([]jobs.BaseJob, 0, len(s.jobs))
 	for _, job := range s.jobs {
 		baseJob := jobs.BaseJob{}
 		err := jsonConvert(job, &baseJob)
 		if err != nil {
-			return internalError(fmt.Errorf("failed to convert job to base job: %w", err))
+			return Response{
+				StatusCode: 400,
+				Body:       fmt.Sprintf("failed to convert job to base job: %s", err),
+			}
 		}
 
 		list = append(list, baseJob)
@@ -141,9 +149,11 @@ func (s *FakeWorkspace) JobsList() (any, int) {
 		return list[i].JobId < list[j].JobId
 	})
 
-	return jobs.ListJobsResponse{
-		Jobs: list,
-	}, http.StatusOK
+	return Response{
+		Body: jobs.ListJobsResponse{
+			Jobs: list,
+		},
+	}
 }
 
 // jsonConvert saves input to a value pointed by output
@@ -163,7 +173,3 @@ func jsonConvert(input, output any) error {
 
 	return nil
 }
-
-func internalError(err error) (string, int) {
-	return fmt.Sprintf("internal error: %s", err), http.StatusInternalServerError
-}
diff --git a/libs/testserver/server.go b/libs/testserver/server.go
index cf4d5aca2..fa15973d7 100644
--- a/libs/testserver/server.go
+++ b/libs/testserver/server.go
@@ -5,14 +5,14 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
+	"reflect"
 	"slices"
 	"strings"
 	"sync"
 
 	"github.com/gorilla/mux"
 
-	"github.com/stretchr/testify/assert"
-
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/databricks-sdk-go/apierr"
 )
@@ -29,10 +29,10 @@ type Server struct {
 	RecordRequests        bool
 	IncludeRequestHeaders []string
 
-	Requests []Request
+	Requests []LoggedRequest
 }
 
-type Request struct {
+type LoggedRequest struct {
 	Headers http.Header `json:"headers,omitempty"`
 	Method  string      `json:"method"`
 	Path    string      `json:"path"`
@@ -40,6 +40,153 @@ type Request struct {
 	RawBody string      `json:"raw_body,omitempty"`
 }
 
+type Request struct {
+	Method    string
+	URL       *url.URL
+	Headers   http.Header
+	Body      []byte
+	Vars      map[string]string
+	Workspace *FakeWorkspace
+}
+
+type Response struct {
+	StatusCode int
+	Headers    http.Header
+	Body       any
+}
+
+type encodedResponse struct {
+	StatusCode int
+	Headers    http.Header
+	Body       []byte
+}
+
+func NewRequest(t testutil.TestingT, r *http.Request, fakeWorkspace *FakeWorkspace) Request {
+	body, err := io.ReadAll(r.Body)
+	if err != nil {
+		t.Fatalf("Failed to read request body: %s", err)
+	}
+
+	return Request{
+		Method:    r.Method,
+		URL:       r.URL,
+		Headers:   r.Header,
+		Body:      body,
+		Vars:      mux.Vars(r),
+		Workspace: fakeWorkspace,
+	}
+}
+
+func normalizeResponse(t testutil.TestingT, resp any) encodedResponse {
+	result := normalizeResponseBody(t, resp)
+	if result.StatusCode == 0 {
+		result.StatusCode = 200
+	}
+	return result
+}
+
+func normalizeResponseBody(t testutil.TestingT, resp any) encodedResponse {
+	if isNil(resp) {
+		t.Errorf("Handler must not return nil")
+		return encodedResponse{StatusCode: 500}
+	}
+
+	respBytes, ok := resp.([]byte)
+	if ok {
+		return encodedResponse{
+			Body:    respBytes,
+			Headers: getHeaders(respBytes),
+		}
+	}
+
+	respString, ok := resp.(string)
+	if ok {
+		return encodedResponse{
+			Body:    []byte(respString),
+			Headers: getHeaders([]byte(respString)),
+		}
+	}
+
+	respStruct, ok := resp.(Response)
+	if ok {
+		if isNil(respStruct.Body) {
+			return encodedResponse{
+				StatusCode: respStruct.StatusCode,
+				Headers:    respStruct.Headers,
+				Body:       []byte{},
+			}
+		}
+
+		bytesVal, isBytes := respStruct.Body.([]byte)
+		if isBytes {
+			return encodedResponse{
+				StatusCode: respStruct.StatusCode,
+				Headers:    respStruct.Headers,
+				Body:       bytesVal,
+			}
+		}
+
+		stringVal, isString := respStruct.Body.(string)
+		if isString {
+			return encodedResponse{
+				StatusCode: respStruct.StatusCode,
+				Headers:    respStruct.Headers,
+				Body:       []byte(stringVal),
+			}
+		}
+
+		respBytes, err := json.MarshalIndent(respStruct.Body, "", "    ")
+		if err != nil {
+			t.Errorf("JSON encoding error: %s", err)
+			return encodedResponse{
+				StatusCode: 500,
+				Body:       []byte("internal error"),
+			}
+		}
+
+		headers := respStruct.Headers
+		if headers == nil {
+			headers = getJsonHeaders()
+		}
+
+		return encodedResponse{
+			StatusCode: respStruct.StatusCode,
+			Headers:    headers,
+			Body:       respBytes,
+		}
+	}
+
+	respBytes, err := json.MarshalIndent(resp, "", "    ")
+	if err != nil {
+		t.Errorf("JSON encoding error: %s", err)
+		return encodedResponse{
+			StatusCode: 500,
+			Body:       []byte("internal error"),
+		}
+	}
+
+	return encodedResponse{
+		Body:    respBytes,
+		Headers: getJsonHeaders(),
+	}
+}
+
+func getJsonHeaders() http.Header {
+	return map[string][]string{
+		"Content-Type": {"application/json"},
+	}
+}
+
+func getHeaders(value []byte) http.Header {
+	if json.Valid(value) {
+		return getJsonHeaders()
+	} else {
+		return map[string][]string{
+			"Content-Type": {"text/plain"},
+		}
+	}
+}
+
 func New(t testutil.TestingT) *Server {
 	router := mux.NewRouter()
 	server := httptest.NewServer(router)
@@ -96,7 +243,7 @@ Response.StatusCode = <response status-code here>
 	return s
 }
 
-type HandlerFunc func(fakeWorkspace *FakeWorkspace, req *http.Request) (resp any, statusCode int)
+type HandlerFunc func(req Request) any
 
 func (s *Server) Handle(method, path string, handler HandlerFunc) {
 	s.Router.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
@@ -117,56 +264,22 @@ func (s *Server) Handle(method, path string, handler HandlerFunc) {
 			fakeWorkspace = s.fakeWorkspaces[token]
 		}
 
-		resp, statusCode := handler(fakeWorkspace, r)
-
+		request := NewRequest(s.t, r, fakeWorkspace)
 		if s.RecordRequests {
-			body, err := io.ReadAll(r.Body)
-			assert.NoError(s.t, err)
-
-			headers := make(http.Header)
-			for k, v := range r.Header {
-				if !slices.Contains(s.IncludeRequestHeaders, k) {
-					continue
-				}
-				for _, vv := range v {
-					headers.Add(k, vv)
-				}
-			}
-
-			req := Request{
-				Headers: headers,
-				Method:  r.Method,
-				Path:    r.URL.Path,
-			}
-
-			if json.Valid(body) {
-				req.Body = json.RawMessage(body)
-			} else {
-				req.RawBody = string(body)
-			}
-
-			s.Requests = append(s.Requests, req)
+			s.Requests = append(s.Requests, getLoggedRequest(request, s.IncludeRequestHeaders))
 		}
 
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(statusCode)
+		respAny := handler(request)
+		resp := normalizeResponse(s.t, respAny)
 
-		var respBytes []byte
-		var err error
-		if respString, ok := resp.(string); ok {
-			respBytes = []byte(respString)
-		} else if respBytes0, ok := resp.([]byte); ok {
-			respBytes = respBytes0
-		} else {
-			respBytes, err = json.MarshalIndent(resp, "", "    ")
-			if err != nil {
-				http.Error(w, err.Error(), http.StatusInternalServerError)
-				return
-			}
+		for k, v := range resp.Headers {
+			w.Header()[k] = v
 		}
 
-		if _, err := w.Write(respBytes); err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
+		w.WriteHeader(resp.StatusCode)
+
+		if _, err := w.Write(resp.Body); err != nil {
+			s.t.Errorf("Failed to write response: %s", err)
 			return
 		}
 	}).Methods(method)
@@ -182,3 +295,43 @@ func getToken(r *http.Request) string {
 
 	return header[len(prefix):]
 }
+
+func getLoggedRequest(req Request, includedHeaders []string) LoggedRequest {
+	result := LoggedRequest{
+		Method:  req.Method,
+		Path:    req.URL.Path,
+		Headers: filterHeaders(req.Headers, includedHeaders),
+	}
+
+	if json.Valid(req.Body) {
+		result.Body = json.RawMessage(req.Body)
+	} else {
+		result.RawBody = string(req.Body)
+	}
+
+	return result
+}
+
+func filterHeaders(h http.Header, includedHeaders []string) http.Header {
+	headers := make(http.Header)
+	for k, v := range h {
+		if !slices.Contains(includedHeaders, k) {
+			continue
+		}
+		headers[k] = v
+	}
+	return headers
+}
+
+func isNil(i any) bool {
+	if i == nil {
+		return true
+	}
+	v := reflect.ValueOf(i)
+	switch v.Kind() {
+	case reflect.Chan, reflect.Func, reflect.Map, reflect.Ptr, reflect.Interface, reflect.Slice:
+		return v.IsNil()
+	default:
+		return false
+	}
+}

From 1dadc794f57b98a833949557f2a61cb44e33146f Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Wed, 12 Feb 2025 14:29:23 +0000
Subject: [PATCH 238/247] [Release] Release v0.241.0 (#2340)

Bundles:
* Added support to generate Git based jobs
([#2304](https://github.com/databricks/cli/pull/2304)).
* Added support for run_as in pipelines
([#2287](https://github.com/databricks/cli/pull/2287)).
* Raise an error when there are multiple local libraries with the same
basename used ([#2297](https://github.com/databricks/cli/pull/2297)).
* Fix env variable for AzureCli local config
([#2248](https://github.com/databricks/cli/pull/2248)).
* Accept JSON files in includes section
([#2265](https://github.com/databricks/cli/pull/2265)).
* Always print warnings and errors; clean up format
([#2213](https://github.com/databricks/cli/pull/2213))

API Changes:
 * Added `databricks account budget-policy` command group.
 * Added `databricks lakeview-embedded` command group.
 * Added `databricks query-execution` command group.
 * Added `databricks account enable-ip-access-lists` command group.
 * Added `databricks redash-config` command group.

OpenAPI commit c72c58f97b950fcb924a90ef164bcb10cfcd5ece (2025-02-03)
Dependency updates:
* Upgrade to TF provider 1.65.1
([#2328](https://github.com/databricks/cli/pull/2328)).
* Bump github.com/hashicorp/terraform-exec from 0.21.0 to 0.22.0
([#2237](https://github.com/databricks/cli/pull/2237)).
* Bump github.com/spf13/pflag from 1.0.5 to 1.0.6
([#2281](https://github.com/databricks/cli/pull/2281)).
* Bump github.com/databricks/databricks-sdk-go from 0.56.1 to 0.57.0
([#2321](https://github.com/databricks/cli/pull/2321)).
* Bump golang.org/x/oauth2 from 0.25.0 to 0.26.0
([#2322](https://github.com/databricks/cli/pull/2322)).
* Bump golang.org/x/term from 0.28.0 to 0.29.0
([#2325](https://github.com/databricks/cli/pull/2325)).
* Bump golang.org/x/text from 0.21.0 to 0.22.0
([#2323](https://github.com/databricks/cli/pull/2323)).
* Bump golang.org/x/mod from 0.22.0 to 0.23.0
([#2324](https://github.com/databricks/cli/pull/2324)).
---
 CHANGELOG.md | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 449c30288..c0202b6a2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,33 @@
 # Version changelog
 
+## [Release] Release v0.241.0
+
+Bundles:
+ * Added support to generate Git based jobs ([#2304](https://github.com/databricks/cli/pull/2304)).
+ * Added support for run_as in pipelines ([#2287](https://github.com/databricks/cli/pull/2287)).
+ * Raise an error when there are multiple local libraries with the same basename used ([#2297](https://github.com/databricks/cli/pull/2297)).
+ * Fix env variable for AzureCli local config ([#2248](https://github.com/databricks/cli/pull/2248)).
+ * Accept JSON files in includes section ([#2265](https://github.com/databricks/cli/pull/2265)).
+ * Always print warnings and errors; clean up format ([#2213](https://github.com/databricks/cli/pull/2213))
+
+API Changes:
+ * Added `databricks account budget-policy` command group.
+ * Added `databricks lakeview-embedded` command group.
+ * Added `databricks query-execution` command group.
+ * Added `databricks account enable-ip-access-lists` command group.
+ * Added `databricks redash-config` command group.
+
+OpenAPI commit c72c58f97b950fcb924a90ef164bcb10cfcd5ece (2025-02-03)
+Dependency updates:
+ * Upgrade to TF provider 1.65.1 ([#2328](https://github.com/databricks/cli/pull/2328)).
+ * Bump github.com/hashicorp/terraform-exec from 0.21.0 to 0.22.0 ([#2237](https://github.com/databricks/cli/pull/2237)).
+ * Bump github.com/spf13/pflag from 1.0.5 to 1.0.6 ([#2281](https://github.com/databricks/cli/pull/2281)).
+ * Bump github.com/databricks/databricks-sdk-go from 0.56.1 to 0.57.0 ([#2321](https://github.com/databricks/cli/pull/2321)).
+ * Bump golang.org/x/oauth2 from 0.25.0 to 0.26.0 ([#2322](https://github.com/databricks/cli/pull/2322)).
+ * Bump golang.org/x/term from 0.28.0 to 0.29.0 ([#2325](https://github.com/databricks/cli/pull/2325)).
+ * Bump golang.org/x/text from 0.21.0 to 0.22.0 ([#2323](https://github.com/databricks/cli/pull/2323)).
+ * Bump golang.org/x/mod from 0.22.0 to 0.23.0 ([#2324](https://github.com/databricks/cli/pull/2324)).
+
 ## [Release] Release v0.240.0
 
 Bundles:

From ac439f8c1a60f7b5dd046caa88ad3ed93d9e0c51 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Wed, 12 Feb 2025 16:14:30 +0000
Subject: [PATCH 239/247] Fix for regression deploying resources with PyPi and
 Maven library types (#2341)

## Changes

The CheckForSameNameLibraries mutator incorrectly assumed all resource
libraries define libraries as paths of the `string` type, but some
libraries, such as PyPi and Maven, define them as objects.

This PR addresses this issue. It was introduced in #2297.

## Tests

Added regression test.
---
 .../artifacts/same_name_libraries/databricks.yml      |  2 ++
 .../bundle/artifacts/same_name_libraries/output.txt   |  2 +-
 bundle/libraries/same_name_libraries.go               | 11 ++++++++---
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/acceptance/bundle/artifacts/same_name_libraries/databricks.yml b/acceptance/bundle/artifacts/same_name_libraries/databricks.yml
index a065bae76..837cd01e8 100644
--- a/acceptance/bundle/artifacts/same_name_libraries/databricks.yml
+++ b/acceptance/bundle/artifacts/same_name_libraries/databricks.yml
@@ -34,6 +34,8 @@ resources:
             package_name: my_default_python
           libraries:
             - whl: ./whl1/dist/*.whl
+            - pypi:
+                package: test_package
         - task_key: task2
           new_cluster: ${var.cluster}
           python_wheel_task:
diff --git a/acceptance/bundle/artifacts/same_name_libraries/output.txt b/acceptance/bundle/artifacts/same_name_libraries/output.txt
index 38cdd43c4..1253d9680 100644
--- a/acceptance/bundle/artifacts/same_name_libraries/output.txt
+++ b/acceptance/bundle/artifacts/same_name_libraries/output.txt
@@ -6,7 +6,7 @@ Error: Duplicate local library name my_default_python-0.0.1-py3-none-any.whl
   at resources.jobs.test.tasks[0].libraries[0].whl
      resources.jobs.test.tasks[1].libraries[0].whl
   in databricks.yml:36:15
-     databricks.yml:43:15
+     databricks.yml:45:15
 
 Local library names must be unique
 
diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go
index 88b96ab54..8de34cfec 100644
--- a/bundle/libraries/same_name_libraries.go
+++ b/bundle/libraries/same_name_libraries.go
@@ -31,13 +31,18 @@ func (c checkForSameNameLibraries) Apply(ctx context.Context, b *bundle.Bundle)
 		var err error
 		for _, pattern := range patterns {
 			v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, lv dyn.Value) (dyn.Value, error) {
-				libPath := lv.MustString()
+				libFullPath, ok := lv.AsString()
+				// If the value is not a string, skip the check because it's not whl or jar type which defines the library
+				// as a string versus PyPi or Maven which defines the library as a map.
+				if !ok {
+					return v, nil
+				}
+
 				// If not local library, skip the check
-				if !IsLibraryLocal(libPath) {
+				if !IsLibraryLocal(libFullPath) {
 					return lv, nil
 				}
 
-				libFullPath := lv.MustString()
 				lib := filepath.Base(libFullPath)
 				// If the same basename was seen already but full path is different
 				// then it's a duplicate. Add the location to the location list.

From 9c90984688060d9c1c889055ab0336536d2a6732 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Wed, 12 Feb 2025 16:17:38 +0000
Subject: [PATCH 240/247] [Release] Release v0.241.1 (#2342)

Bundles:
* Fix for regression deploying resources with PyPi and Maven library
types ([#2341](https://github.com/databricks/cli/pull/2341)).
---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c0202b6a2..d3df510b7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,10 @@
 # Version changelog
 
+## [Release] Release v0.241.1
+
+Bundles:
+ * Fix for regression deploying resources with PyPi and Maven library types ([#2341](https://github.com/databricks/cli/pull/2341)).
+
 ## [Release] Release v0.241.0
 
 Bundles:

From 96302c7415331e925fc9d6056b3e34923e84f64f Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Wed, 12 Feb 2025 20:05:49 +0100
Subject: [PATCH 241/247] Revert changes related to basename check for local
 libraries (#2345)

## Changes

These changes break the use of non-local libraries (such as PyPI
libraries).

This reverts the set so we can cut a patch release and take a closer
look later.

Original PRs are #2297 and #2341.

Issue reported in #2343.

## Tests

Manually confirmed that a bundle with PyPI package in libraries now
deploys fine.
---
 .../same_name_libraries/databricks.yml        |  52 --------
 .../artifacts/same_name_libraries/output.txt  |  14 --
 .../artifacts/same_name_libraries/script      |   2 -
 .../artifacts/same_name_libraries/test.toml   |   0
 .../same_name_libraries/whl1/setup.py         |  36 ------
 .../whl1/src/my_default_python/__init__.py    |   1 -
 .../whl1/src/my_default_python/main.py        |   1 -
 .../same_name_libraries/whl2/setup.py         |  36 ------
 .../whl2/src/my_default_python/__init__.py    |   1 -
 .../whl2/src/my_default_python/main.py        |   1 -
 bundle/libraries/expand_glob_references.go    |   2 +-
 bundle/libraries/same_name_libraries.go       | 102 ---------------
 bundle/libraries/same_name_libraries_test.go  | 121 ------------------
 bundle/phases/deploy.go                       |   5 -
 14 files changed, 1 insertion(+), 373 deletions(-)
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/databricks.yml
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/output.txt
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/script
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/test.toml
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py
 delete mode 100644 bundle/libraries/same_name_libraries.go
 delete mode 100644 bundle/libraries/same_name_libraries_test.go

diff --git a/acceptance/bundle/artifacts/same_name_libraries/databricks.yml b/acceptance/bundle/artifacts/same_name_libraries/databricks.yml
deleted file mode 100644
index 837cd01e8..000000000
--- a/acceptance/bundle/artifacts/same_name_libraries/databricks.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-bundle:
-  name: same_name_libraries
-
-variables:
-  cluster:
-    default:
-      spark_version: 15.4.x-scala2.12
-      node_type_id: i3.xlarge
-      data_security_mode: SINGLE_USER
-      num_workers: 0
-      spark_conf:
-          spark.master: "local[*, 4]"
-          spark.databricks.cluster.profile: singleNode
-      custom_tags:
-        ResourceClass: SingleNode
-
-artifacts:
-  whl1:
-    type: whl
-    path: ./whl1
-  whl2:
-    type: whl
-    path: ./whl2
-
-resources:
-  jobs:
-    test:
-      name: "test"
-      tasks:
-        - task_key: task1
-          new_cluster: ${var.cluster}
-          python_wheel_task:
-            entry_point: main
-            package_name: my_default_python
-          libraries:
-            - whl: ./whl1/dist/*.whl
-            - pypi:
-                package: test_package
-        - task_key: task2
-          new_cluster: ${var.cluster}
-          python_wheel_task:
-            entry_point: main
-            package_name: my_default_python
-          libraries:
-            - whl: ./whl2/dist/*.whl
-        - task_key: task3
-          new_cluster: ${var.cluster}
-          python_wheel_task:
-            entry_point: main
-            package_name: my_default_python
-          libraries:
-            - whl: ./whl1/dist/*.whl
diff --git a/acceptance/bundle/artifacts/same_name_libraries/output.txt b/acceptance/bundle/artifacts/same_name_libraries/output.txt
deleted file mode 100644
index 1253d9680..000000000
--- a/acceptance/bundle/artifacts/same_name_libraries/output.txt
+++ /dev/null
@@ -1,14 +0,0 @@
-
->>> errcode [CLI] bundle deploy
-Building whl1...
-Building whl2...
-Error: Duplicate local library name my_default_python-0.0.1-py3-none-any.whl
-  at resources.jobs.test.tasks[0].libraries[0].whl
-     resources.jobs.test.tasks[1].libraries[0].whl
-  in databricks.yml:36:15
-     databricks.yml:45:15
-
-Local library names must be unique
-
-
-Exit code: 1
diff --git a/acceptance/bundle/artifacts/same_name_libraries/script b/acceptance/bundle/artifacts/same_name_libraries/script
deleted file mode 100644
index 6c899df07..000000000
--- a/acceptance/bundle/artifacts/same_name_libraries/script
+++ /dev/null
@@ -1,2 +0,0 @@
-trace errcode $CLI bundle deploy
-rm -rf whl1 whl2
diff --git a/acceptance/bundle/artifacts/same_name_libraries/test.toml b/acceptance/bundle/artifacts/same_name_libraries/test.toml
deleted file mode 100644
index e69de29bb..000000000
diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py
deleted file mode 100644
index 1afaf3a4f..000000000
--- a/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py
+++ /dev/null
@@ -1,36 +0,0 @@
-"""
-setup.py configuration script describing how to build and package this project.
-
-This file is primarily used by the setuptools library and typically should not
-be executed directly. See README.md for how to deploy, test, and run
-the my_default_python project.
-"""
-
-from setuptools import setup, find_packages
-
-import sys
-
-sys.path.append("./src")
-
-import my_default_python
-
-setup(
-    name="my_default_python",
-    version=my_default_python.__version__,
-    url="https://databricks.com",
-    author="[USERNAME]",
-    description="wheel file based on my_default_python/src",
-    packages=find_packages(where="./src"),
-    package_dir={"": "src"},
-    entry_points={
-        "packages": [
-            "main=my_default_python.main:main",
-        ],
-    },
-    install_requires=[
-        # Dependencies in case the output wheel file is used as a library dependency.
-        # For defining dependencies, when this package is used in Databricks, see:
-        # https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
-        "setuptools"
-    ],
-)
diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py
deleted file mode 100644
index f102a9cad..000000000
--- a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-__version__ = "0.0.1"
diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py
deleted file mode 100644
index 11b15b1a4..000000000
--- a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py
+++ /dev/null
@@ -1 +0,0 @@
-print("hello")
diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py
deleted file mode 100644
index 1afaf3a4f..000000000
--- a/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py
+++ /dev/null
@@ -1,36 +0,0 @@
-"""
-setup.py configuration script describing how to build and package this project.
-
-This file is primarily used by the setuptools library and typically should not
-be executed directly. See README.md for how to deploy, test, and run
-the my_default_python project.
-"""
-
-from setuptools import setup, find_packages
-
-import sys
-
-sys.path.append("./src")
-
-import my_default_python
-
-setup(
-    name="my_default_python",
-    version=my_default_python.__version__,
-    url="https://databricks.com",
-    author="[USERNAME]",
-    description="wheel file based on my_default_python/src",
-    packages=find_packages(where="./src"),
-    package_dir={"": "src"},
-    entry_points={
-        "packages": [
-            "main=my_default_python.main:main",
-        ],
-    },
-    install_requires=[
-        # Dependencies in case the output wheel file is used as a library dependency.
-        # For defining dependencies, when this package is used in Databricks, see:
-        # https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
-        "setuptools"
-    ],
-)
diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py
deleted file mode 100644
index f102a9cad..000000000
--- a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-__version__ = "0.0.1"
diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py
deleted file mode 100644
index 11b15b1a4..000000000
--- a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py
+++ /dev/null
@@ -1 +0,0 @@
-print("hello")
diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go
index 7a808f627..bb1905045 100644
--- a/bundle/libraries/expand_glob_references.go
+++ b/bundle/libraries/expand_glob_references.go
@@ -92,7 +92,7 @@ func expandLibraries(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostic
 
 		for _, match := range matches {
 			output = append(output, dyn.NewValue(map[string]dyn.Value{
-				libType: dyn.NewValue(match, lib.Locations()),
+				libType: dyn.V(match),
 			}, lib.Locations()))
 		}
 	}
diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go
deleted file mode 100644
index 8de34cfec..000000000
--- a/bundle/libraries/same_name_libraries.go
+++ /dev/null
@@ -1,102 +0,0 @@
-package libraries
-
-import (
-	"context"
-	"path/filepath"
-
-	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/dyn"
-)
-
-type checkForSameNameLibraries struct{}
-
-var patterns = []dyn.Pattern{
-	taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.AnyKey()),
-	forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.AnyKey()),
-	envDepsPattern.Append(dyn.AnyIndex()),
-}
-
-type libData struct {
-	fullPath  string
-	locations []dyn.Location
-	paths     []dyn.Path
-}
-
-func (c checkForSameNameLibraries) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-	var diags diag.Diagnostics
-	libs := make(map[string]*libData)
-
-	err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
-		var err error
-		for _, pattern := range patterns {
-			v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, lv dyn.Value) (dyn.Value, error) {
-				libFullPath, ok := lv.AsString()
-				// If the value is not a string, skip the check because it's not whl or jar type which defines the library
-				// as a string versus PyPi or Maven which defines the library as a map.
-				if !ok {
-					return v, nil
-				}
-
-				// If not local library, skip the check
-				if !IsLibraryLocal(libFullPath) {
-					return lv, nil
-				}
-
-				lib := filepath.Base(libFullPath)
-				// If the same basename was seen already but full path is different
-				// then it's a duplicate. Add the location to the location list.
-				lp, ok := libs[lib]
-				if !ok {
-					libs[lib] = &libData{
-						fullPath:  libFullPath,
-						locations: []dyn.Location{lv.Location()},
-						paths:     []dyn.Path{p},
-					}
-				} else if lp.fullPath != libFullPath {
-					lp.locations = append(lp.locations, lv.Location())
-					lp.paths = append(lp.paths, p)
-				}
-
-				return lv, nil
-			})
-			if err != nil {
-				return dyn.InvalidValue, err
-			}
-		}
-
-		if err != nil {
-			return dyn.InvalidValue, err
-		}
-
-		return v, nil
-	})
-
-	// Iterate over all the libraries and check if there are any duplicates.
-	// Duplicates will have more than one location.
-	// If there are duplicates, add a diagnostic.
-	for lib, lv := range libs {
-		if len(lv.locations) > 1 {
-			diags = append(diags, diag.Diagnostic{
-				Severity:  diag.Error,
-				Summary:   "Duplicate local library name " + lib,
-				Detail:    "Local library names must be unique",
-				Locations: lv.locations,
-				Paths:     lv.paths,
-			})
-		}
-	}
-	if err != nil {
-		diags = diags.Extend(diag.FromErr(err))
-	}
-
-	return diags
-}
-
-func (c checkForSameNameLibraries) Name() string {
-	return "CheckForSameNameLibraries"
-}
-
-func CheckForSameNameLibraries() bundle.Mutator {
-	return checkForSameNameLibraries{}
-}
diff --git a/bundle/libraries/same_name_libraries_test.go b/bundle/libraries/same_name_libraries_test.go
deleted file mode 100644
index 42c38773b..000000000
--- a/bundle/libraries/same_name_libraries_test.go
+++ /dev/null
@@ -1,121 +0,0 @@
-package libraries
-
-import (
-	"context"
-	"testing"
-
-	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/bundle/config"
-	"github.com/databricks/cli/bundle/config/resources"
-	"github.com/databricks/cli/bundle/internal/bundletest"
-	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/dyn"
-	"github.com/databricks/databricks-sdk-go/service/compute"
-	"github.com/databricks/databricks-sdk-go/service/jobs"
-	"github.com/stretchr/testify/require"
-)
-
-func TestSameNameLibraries(t *testing.T) {
-	b := &bundle.Bundle{
-		Config: config.Root{
-			Resources: config.Resources{
-				Jobs: map[string]*resources.Job{
-					"test": {
-						JobSettings: &jobs.JobSettings{
-							Tasks: []jobs.Task{
-								{
-									Libraries: []compute.Library{
-										{
-											Whl: "full/path/test.whl",
-										},
-									},
-								},
-								{
-									Libraries: []compute.Library{
-										{
-											Whl: "other/path/test.whl",
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-
-	bundletest.SetLocation(b, "resources.jobs.test.tasks[0]", []dyn.Location{
-		{File: "databricks.yml", Line: 10, Column: 1},
-	})
-	bundletest.SetLocation(b, "resources.jobs.test.tasks[1]", []dyn.Location{
-		{File: "databricks.yml", Line: 20, Column: 1},
-	})
-
-	diags := bundle.Apply(context.Background(), b, CheckForSameNameLibraries())
-	require.Len(t, diags, 1)
-	require.Equal(t, diag.Error, diags[0].Severity)
-	require.Equal(t, "Duplicate local library name test.whl", diags[0].Summary)
-	require.Equal(t, []dyn.Location{
-		{File: "databricks.yml", Line: 10, Column: 1},
-		{File: "databricks.yml", Line: 20, Column: 1},
-	}, diags[0].Locations)
-
-	paths := make([]string, 0)
-	for _, p := range diags[0].Paths {
-		paths = append(paths, p.String())
-	}
-	require.Equal(t, []string{
-		"resources.jobs.test.tasks[0].libraries[0].whl",
-		"resources.jobs.test.tasks[1].libraries[0].whl",
-	}, paths)
-}
-
-func TestSameNameLibrariesWithUniqueLibraries(t *testing.T) {
-	b := &bundle.Bundle{
-		Config: config.Root{
-			Resources: config.Resources{
-				Jobs: map[string]*resources.Job{
-					"test": {
-						JobSettings: &jobs.JobSettings{
-							Tasks: []jobs.Task{
-								{
-									Libraries: []compute.Library{
-										{
-											Whl: "full/path/test-0.1.1.whl",
-										},
-
-										{
-											Whl: "cowsay",
-										},
-									},
-								},
-								{
-									Libraries: []compute.Library{
-										{
-											Whl: "other/path/test-0.1.0.whl",
-										},
-
-										{
-											Whl: "cowsay",
-										},
-									},
-								},
-								{
-									Libraries: []compute.Library{
-										{
-											Whl: "full/path/test-0.1.1.whl", // Use the same library as the first task
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-
-	diags := bundle.Apply(context.Background(), b, CheckForSameNameLibraries())
-	require.Empty(t, diags)
-}
diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go
index 2e9211a7e..c6ec04962 100644
--- a/bundle/phases/deploy.go
+++ b/bundle/phases/deploy.go
@@ -155,11 +155,6 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator {
 				mutator.ValidateGitDetails(),
 				artifacts.CleanUp(),
 				libraries.ExpandGlobReferences(),
-				// libraries.CheckForSameNameLibraries() needs to be run after we expand glob references so we
-				// know what are the actual library paths.
-				// libraries.ExpandGlobReferences() has to be run after the libraries are built and thus this
-				// mutator is part of the deploy step rather than validate.
-				libraries.CheckForSameNameLibraries(),
 				libraries.Upload(),
 				trampoline.TransformWheelTask(),
 				files.Upload(outputHandler),

From a20894b1f2e5569f2dd73df13f973806d8700f93 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Wed, 12 Feb 2025 20:21:48 +0100
Subject: [PATCH 242/247] [Release] Release v0.241.2 (#2346)

This is a bugfix release to address an issue where jobs with tasks with
a
libraries section with PyPI packages could not be deployed.

Bundles:
* Revert changes related to basename check for local libraries
([#2345](https://github.com/databricks/cli/pull/2345)).
---
 CHANGELOG.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d3df510b7..23c696ab7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,13 @@
 # Version changelog
 
+## [Release] Release v0.241.2
+
+This is a bugfix release to address an issue where jobs with tasks with a
+libraries section with PyPI packages could not be deployed.
+
+Bundles:
+ * Revert changes related to basename check for local libraries ([#2345](https://github.com/databricks/cli/pull/2345)).
+
 ## [Release] Release v0.241.1
 
 Bundles:

From fac9bcf1afe6365dfe49c81ae9ea92135d660de9 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 13 Feb 2025 08:26:22 +0100
Subject: [PATCH 243/247] acc: Set X-Databricks-Org-Id on scim/v2/Me endpoint
 (#2349)

This is needed for b.WorkspaceClient().CurrentWorkspaceID(ctx) which is
used by initialize_urls.go mutator ("bundle summary") #2316

It also also needed for to call serverless detection endpoint #2348

Builds on top of #2338
---
 acceptance/server_test.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/acceptance/server_test.go b/acceptance/server_test.go
index f73872e0b..4fc3108d2 100644
--- a/acceptance/server_test.go
+++ b/acceptance/server_test.go
@@ -63,7 +63,10 @@ func AddHandlers(server *testserver.Server) {
 	})
 
 	server.Handle("GET", "/api/2.0/preview/scim/v2/Me", func(req testserver.Request) any {
-		return testUser
+		return testserver.Response{
+			Headers: map[string][]string{"X-Databricks-Org-Id": {"900800700600"}},
+			Body:    testUser,
+		}
 	})
 
 	server.Handle("GET", "/api/2.0/workspace/get-status", func(req testserver.Request) any {

From 2d0963661136d5901f6acf8fa7c296ff90b9ca8d Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 13 Feb 2025 08:31:04 +0100
Subject: [PATCH 244/247] acc: do not show diff for missing output file (#2350)

It's not interesting since it just dumps what is in the repo. This is
especially annoying with bundle/templates tests with a lot of files.
---
 acceptance/acceptance_test.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 85c345032..d99ad2991 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -402,8 +402,7 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN
 
 	// The test did not produce an expected output file.
 	if okRef && !okNew {
-		t.Errorf("Missing output file: %s\npathRef: %s\npathNew: %s", relPath, pathRef, pathNew)
-		testdiff.AssertEqualTexts(t, pathRef, pathNew, valueRef, valueNew)
+		t.Errorf("Missing output file: %s", relPath)
 		if testdiff.OverwriteMode {
 			t.Logf("Removing output file: %s", relPath)
 			require.NoError(t, os.Remove(pathRef))

From c0a56a93fb5ad472d9ba88c6e4fd1cf14973ec70 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 14 Feb 2025 12:02:12 +0100
Subject: [PATCH 245/247] acc: add a helper to diff with replacements (#2352)

## Changes

diff.py is like "diff -r -U2" but it applies replacements first to the
argument.

This allows comparing different output files and directories but ignore
differences that are going to be replaced by placeholders.

This is useful for tests that record large amount of files, specifically
"bundle init" with standard templates. In those tests, changing one
parameter results in a small diff so recording the full directory is not
helpful, because it's hard to see what changed there. I'm using it in
implementation of serverless mode for templates that need it: #2348 The
serverless templates are slightly different from classic, capturing the
diff helps to see exactly where.

Related small changes:
- Add [TESTROOT] replacement for absolute path to acceptance directory
in git repo.
- Add $TESTDIR env var for absolute path to a given test in git repo.

## Tests
- New test acceptance/selftest/diff to test the helper.
- Via #2348 which makes use of this feature.
---
 acceptance/acceptance_test.go                 | 20 +++++++
 acceptance/bin/diff.py                        | 56 +++++++++++++++++++
 acceptance/selftest/diff/out_dir_a/output.txt |  7 +++
 acceptance/selftest/diff/out_dir_b/output.txt |  7 +++
 acceptance/selftest/diff/output.txt           | 13 +++++
 acceptance/selftest/diff/script               | 17 ++++++
 acceptance/selftest/test.toml                 |  1 +
 7 files changed, 121 insertions(+)
 create mode 100755 acceptance/bin/diff.py
 create mode 100644 acceptance/selftest/diff/out_dir_a/output.txt
 create mode 100644 acceptance/selftest/diff/out_dir_b/output.txt
 create mode 100644 acceptance/selftest/diff/output.txt
 create mode 100644 acceptance/selftest/diff/script
 create mode 100644 acceptance/selftest/test.toml

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index d99ad2991..c7b1151ab 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -57,6 +57,8 @@ const (
 	CleanupScript    = "script.cleanup"
 	PrepareScript    = "script.prepare"
 	MaxFileSize      = 100_000
+	// Filename to save replacements to (used by diff.py)
+	ReplsFile = "repls.json"
 )
 
 var Scripts = map[string]bool{
@@ -65,6 +67,10 @@ var Scripts = map[string]bool{
 	PrepareScript:    true,
 }
 
+var Ignored = map[string]bool{
+	ReplsFile: true,
+}
+
 func TestAccept(t *testing.T) {
 	testAccept(t, InprocessMode, SingleTest)
 }
@@ -152,6 +158,8 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	testdiff.PrepareReplacementSdkVersion(t, &repls)
 	testdiff.PrepareReplacementsGoVersion(t, &repls)
 
+	repls.SetPath(cwd, "[TESTROOT]")
+
 	repls.Repls = append(repls.Repls, testdiff.Replacement{Old: regexp.MustCompile("dbapi[0-9a-f]+"), New: "[DATABRICKS_TOKEN]"})
 
 	testDirs := getTests(t)
@@ -310,6 +318,11 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 	// User replacements come last:
 	repls.Repls = append(repls.Repls, config.Repls...)
 
+	// Save replacements to temp test directory so that it can be read by diff.py
+	replsJson, err := json.MarshalIndent(repls.Repls, "", "  ")
+	require.NoError(t, err)
+	testutil.WriteFile(t, filepath.Join(tmpDir, ReplsFile), string(replsJson))
+
 	if coverDir != "" {
 		// Creating individual coverage directory for each test, because writing to the same one
 		// results in sporadic failures like this one (only if tests are running in parallel):
@@ -320,6 +333,10 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir)
 	}
 
+	absDir, err := filepath.Abs(dir)
+	require.NoError(t, err)
+	cmd.Env = append(cmd.Env, "TESTDIR="+absDir)
+
 	// Write combined output to a file
 	out, err := os.Create(filepath.Join(tmpDir, "output.txt"))
 	require.NoError(t, err)
@@ -368,6 +385,9 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		if _, ok := outputs[relPath]; ok {
 			continue
 		}
+		if _, ok := Ignored[relPath]; ok {
+			continue
+		}
 		unexpected = append(unexpected, relPath)
 		if strings.HasPrefix(relPath, "out") {
 			// We have a new file starting with "out"
diff --git a/acceptance/bin/diff.py b/acceptance/bin/diff.py
new file mode 100755
index 000000000..0a91d57ce
--- /dev/null
+++ b/acceptance/bin/diff.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+"""This script implements "diff -r -U2 dir1 dir2" but applies replacements first"""
+
+import sys
+import difflib
+import json
+import re
+from pathlib import Path
+
+
+def replaceAll(patterns, s):
+    for comp, new in patterns:
+        s = comp.sub(new, s)
+    return s
+
+
+def main():
+    d1, d2 = sys.argv[1:]
+    d1, d2 = Path(d1), Path(d2)
+
+    with open("repls.json") as f:
+        repls = json.load(f)
+
+    patterns = []
+    for r in repls:
+        try:
+            c = re.compile(r["Old"])
+            patterns.append((c, r["New"]))
+        except re.error as e:
+            print(f"Regex error for pattern {r}: {e}", file=sys.stderr)
+
+    files1 = [str(p.relative_to(d1)) for p in d1.rglob("*") if p.is_file()]
+    files2 = [str(p.relative_to(d2)) for p in d2.rglob("*") if p.is_file()]
+
+    set1 = set(files1)
+    set2 = set(files2)
+
+    for f in sorted(set1 | set2):
+        p1 = d1 / f
+        p2 = d2 / f
+        if f not in set2:
+            print(f"Only in {d1}: {f}")
+        elif f not in set1:
+            print(f"Only in {d2}: {f}")
+        else:
+            a = [replaceAll(patterns, x) for x in p1.read_text().splitlines(True)]
+            b = [replaceAll(patterns, x) for x in p2.read_text().splitlines(True)]
+            if a != b:
+                p1_str = p1.as_posix()
+                p2_str = p2.as_posix()
+                for line in difflib.unified_diff(a, b, p1_str, p2_str, "", "", 2):
+                    print(line, end="")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/acceptance/selftest/diff/out_dir_a/output.txt b/acceptance/selftest/diff/out_dir_a/output.txt
new file mode 100644
index 000000000..303c1867b
--- /dev/null
+++ b/acceptance/selftest/diff/out_dir_a/output.txt
@@ -0,0 +1,7 @@
+Hello!
+{
+    "id": "[USERID]",
+    "userName": "[USERNAME]"
+}
+
+Footer
\ No newline at end of file
diff --git a/acceptance/selftest/diff/out_dir_b/output.txt b/acceptance/selftest/diff/out_dir_b/output.txt
new file mode 100644
index 000000000..f4f01af13
--- /dev/null
+++ b/acceptance/selftest/diff/out_dir_b/output.txt
@@ -0,0 +1,7 @@
+Hello!
+{
+    "id": "[UUID]",
+    "userName": "[USERNAME]"
+}
+
+Footer
\ No newline at end of file
diff --git a/acceptance/selftest/diff/output.txt b/acceptance/selftest/diff/output.txt
new file mode 100644
index 000000000..aef99f1e3
--- /dev/null
+++ b/acceptance/selftest/diff/output.txt
@@ -0,0 +1,13 @@
+
+>>> diff.py out_dir_a out_dir_b
+Only in out_dir_a: only_in_a
+Only in out_dir_b: only_in_b
+--- out_dir_a/output.txt
++++ out_dir_b/output.txt
+@@ -1,5 +1,5 @@
+ Hello!
+ {
+-    "id": "[USERID]",
++    "id": "[UUID]",
+     "userName": "[USERNAME]"
+ }
diff --git a/acceptance/selftest/diff/script b/acceptance/selftest/diff/script
new file mode 100644
index 000000000..a7b8706e6
--- /dev/null
+++ b/acceptance/selftest/diff/script
@@ -0,0 +1,17 @@
+mkdir out_dir_a
+mkdir out_dir_b
+
+touch out_dir_a/only_in_a
+touch out_dir_b/only_in_b
+
+echo Hello! >> out_dir_a/output.txt
+echo Hello! >> out_dir_b/output.txt
+
+curl -s $DATABRICKS_HOST/api/2.0/preview/scim/v2/Me >> out_dir_a/output.txt
+printf "\n\nFooter" >> out_dir_a/output.txt
+printf '{\n    "id": "7d639bad-ac6d-4e6f-abd7-9522a86b0239",\n    "userName": "[USERNAME]"\n}\n\nFooter' >> out_dir_b/output.txt
+
+# Unlike regular diff, diff.py will apply replacements first before doing the comparison
+errcode trace diff.py out_dir_a out_dir_b
+
+rm out_dir_a/only_in_a out_dir_b/only_in_b
diff --git a/acceptance/selftest/test.toml b/acceptance/selftest/test.toml
new file mode 100644
index 000000000..b76e712fb
--- /dev/null
+++ b/acceptance/selftest/test.toml
@@ -0,0 +1 @@
+LocalOnly = true

From bc30d440972efe1b280d54d3afe5d7593115c978 Mon Sep 17 00:00:00 2001
From: "Lennart Kats (databricks)" <lennart.kats@databricks.com>
Date: Mon, 17 Feb 2025 13:38:03 +0100
Subject: [PATCH 246/247] Provide instructions for testing in the
 default-python template (#2355)

## Changes
Adds instructions for testing to the default-python template.

## Tests
- Unit & acceptance tests.
---
 .../default-python/output/my_default_python/README.md  | 10 ++++++----
 .../template/{{.project_name}}/README.md.tmpl          | 10 ++++++++--
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/README.md b/acceptance/bundle/templates/default-python/output/my_default_python/README.md
index 97d7d7949..10f570bf4 100644
--- a/acceptance/bundle/templates/default-python/output/my_default_python/README.md
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/README.md
@@ -37,10 +37,12 @@ The 'my_default_python' project was generated by using the default-python templa
    ```
    $ databricks bundle run
    ```
-
-6. Optionally, install developer tools such as the Databricks extension for Visual Studio Code from
-   https://docs.databricks.com/dev-tools/vscode-ext.html. Or read the "getting started" documentation for
-   **Databricks Connect** for instructions on running the included Python code from a different IDE.
+6. Optionally, install the Databricks extension for Visual Studio code for local development from
+   https://docs.databricks.com/dev-tools/vscode-ext.html. It can configure your
+   virtual environment and setup Databricks Connect for running unit tests locally.
+   When not using these tools, consult your development environment's documentation
+   and/or the documentation for Databricks Connect for manually setting up your environment
+   (https://docs.databricks.com/en/dev-tools/databricks-connect/python/index.html).
 
 7. For documentation on the Databricks asset bundles format used
    for this project, and for CI/CD configuration, see
diff --git a/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl
index 53847a9c9..b8811fa3e 100644
--- a/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl
+++ b/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl
@@ -38,10 +38,16 @@ The '{{.project_name}}' project was generated by using the default-python templa
    $ databricks bundle run
    ```
 
+{{- if (eq .include_python "no") }}
 6. Optionally, install developer tools such as the Databricks extension for Visual Studio Code from
    https://docs.databricks.com/dev-tools/vscode-ext.html.
-{{- if (eq .include_python "yes") }} Or read the "getting started" documentation for
-   **Databricks Connect** for instructions on running the included Python code from a different IDE.
+{{- else }}
+6. Optionally, install the Databricks extension for Visual Studio code for local development from
+   https://docs.databricks.com/dev-tools/vscode-ext.html. It can configure your
+   virtual environment and setup Databricks Connect for running unit tests locally.
+   When not using these tools, consult your development environment's documentation
+   and/or the documentation for Databricks Connect for manually setting up your environment
+   (https://docs.databricks.com/en/dev-tools/databricks-connect/python/index.html).
 {{- end}}
 
 7. For documentation on the Databricks asset bundles format used

From 874a05a27b750f2f732436d2b43d90997dc6b32c Mon Sep 17 00:00:00 2001
From: Ilya Kuznetsov <ilya.kuznetsov@databricks.com>
Date: Tue, 18 Feb 2025 17:12:49 +0100
Subject: [PATCH 247/247] Add escaping for links and headers in docsgen (#2330)

## Changes

To avoid build warnings and errors in docs build we need to escape
symbols that are treated as syntax elements

## Tests
<!-- How is this tested? -->
---
 bundle/docsgen/markdown.go             |  23 +-
 bundle/docsgen/markdown_test.go        |  42 ++
 bundle/docsgen/nodes.go                |  10 +-
 bundle/docsgen/output/reference.md     |  90 +--
 bundle/docsgen/output/resources.md     | 940 +++++++++++++------------
 bundle/docsgen/testdata/anchors.md     |  28 +
 bundle/internal/schema/annotations.yml |  10 -
 bundle/internal/schema/main_test.go    |  20 +
 8 files changed, 651 insertions(+), 512 deletions(-)
 create mode 100644 bundle/docsgen/markdown_test.go
 create mode 100644 bundle/docsgen/testdata/anchors.md

diff --git a/bundle/docsgen/markdown.go b/bundle/docsgen/markdown.go
index 6e3b42b65..b711aa0e8 100644
--- a/bundle/docsgen/markdown.go
+++ b/bundle/docsgen/markdown.go
@@ -12,10 +12,11 @@ func buildMarkdown(nodes []rootNode, outputFile, header string) error {
 	m = m.PlainText(header)
 	for _, node := range nodes {
 		m = m.LF()
+		title := escapeBrackets(node.Title)
 		if node.TopLevel {
-			m = m.H2(node.Title)
+			m = m.H2(title)
 		} else {
-			m = m.H3(node.Title)
+			m = m.H3(title)
 		}
 		m = m.LF()
 
@@ -93,7 +94,23 @@ func formatDescription(a attributeNode) string {
 		} else if s != "" {
 			s += ". "
 		}
-		s += fmt.Sprintf("See [_](#%s).", a.Link)
+		s += fmt.Sprintf("See [_](#%s).", cleanAnchor(a.Link))
 	}
 	return s
 }
+
+// Docs framework does not allow special characters in anchor links and strip them out by default
+// We need to clean them up to make sure the links pass the validation
+func cleanAnchor(s string) string {
+	s = strings.ReplaceAll(s, "<", "")
+	s = strings.ReplaceAll(s, ">", "")
+	s = strings.ReplaceAll(s, ".", "")
+
+	return s
+}
+
+func escapeBrackets(s string) string {
+	s = strings.ReplaceAll(s, "<", "\\<")
+	s = strings.ReplaceAll(s, ">", "\\>")
+	return s
+}
diff --git a/bundle/docsgen/markdown_test.go b/bundle/docsgen/markdown_test.go
new file mode 100644
index 000000000..d4f32230e
--- /dev/null
+++ b/bundle/docsgen/markdown_test.go
@@ -0,0 +1,42 @@
+package main
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/cli/internal/testutil"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBuildMarkdownAnchors(t *testing.T) {
+	nodes := []rootNode{
+		{
+			Title:       "some_field",
+			TopLevel:    true,
+			Type:        "Map",
+			Description: "This is a description",
+			Attributes: []attributeNode{
+				{
+					Title:       "my_attribute",
+					Type:        "Map",
+					Description: "Desc with link",
+					Link:        "some_field.<name>.my_attribute",
+				},
+			},
+		},
+		{
+			Title:       "some_field.<name>.my_attribute",
+			TopLevel:    false,
+			Type:        "Boolean",
+			Description: "Another description",
+		},
+	}
+	tmpDir := t.TempDir()
+	path := filepath.Join(tmpDir, "output.md")
+
+	err := buildMarkdown(nodes, path, "Header")
+	require.NoError(t, err)
+
+	expected := testutil.ReadFile(t, "testdata/anchors.md")
+	testutil.AssertFileContents(t, path, expected)
+}
diff --git a/bundle/docsgen/nodes.go b/bundle/docsgen/nodes.go
index 61d2c21cc..6645e9ccc 100644
--- a/bundle/docsgen/nodes.go
+++ b/bundle/docsgen/nodes.go
@@ -65,7 +65,7 @@ func buildNodes(s jsonschema.Schema, refs map[string]*jsonschema.Schema, ownFiel
 		v = resolveRefs(v, refs)
 		node := rootNode{
 			Title:       k,
-			Description: getDescription(v, item.topLevel),
+			Description: getDescription(v),
 			TopLevel:    item.topLevel,
 			Example:     getExample(v),
 			Type:        getHumanReadableType(v.Type),
@@ -78,7 +78,7 @@ func buildNodes(s jsonschema.Schema, refs map[string]*jsonschema.Schema, ownFiel
 
 		mapValueType := getMapValueType(v, refs)
 		if mapValueType != nil {
-			d := getDescription(mapValueType, true)
+			d := getDescription(mapValueType)
 			if d != "" {
 				node.Description = d
 			}
@@ -174,7 +174,7 @@ func getAttributes(props, refs map[string]*jsonschema.Schema, ownFields map[stri
 		attributes = append(attributes, attributeNode{
 			Title:       k,
 			Type:        typeString,
-			Description: getDescription(v, true),
+			Description: getDescription(v),
 			Link:        reference,
 		})
 	}
@@ -184,8 +184,8 @@ func getAttributes(props, refs map[string]*jsonschema.Schema, ownFields map[stri
 	return attributes
 }
 
-func getDescription(s *jsonschema.Schema, allowMarkdown bool) string {
-	if allowMarkdown && s.MarkdownDescription != "" {
+func getDescription(s *jsonschema.Schema) string {
+	if s.MarkdownDescription != "" {
 		return s.MarkdownDescription
 	}
 	return s.Description
diff --git a/bundle/docsgen/output/reference.md b/bundle/docsgen/output/reference.md
index 8a89d354b..0de3c6f2b 100644
--- a/bundle/docsgen/output/reference.md
+++ b/bundle/docsgen/output/reference.md
@@ -43,7 +43,7 @@ artifacts:
   
    * - `files`
      - Sequence
-     - The source files for the artifact. See [_](#artifacts.<name>.files).
+     - The source files for the artifact. See [_](#artifactsnamefiles).
   
    * - `path`
      - String
@@ -64,7 +64,7 @@ artifacts:
     path: .
 ```
   
-### artifacts.<name>.files
+### artifacts.\<name\>.files
   
 **`Type: Sequence`**
   
@@ -113,11 +113,11 @@ The bundle attributes when deploying to this target,
   
    * - `deployment`
      - Map
-     - The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). See [_](#bundle.deployment).
+     - The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). See [_](#bundledeployment).
   
    * - `git`
      - Map
-     - The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). See [_](#bundle.git).
+     - The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). See [_](#bundlegit).
   
    * - `name`
      - String
@@ -132,7 +132,7 @@ The bundle attributes when deploying to this target,
   
 **`Type: Map`**
   
-The definition of the bundle deployment
+The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md).
   
   
   
@@ -149,7 +149,7 @@ The definition of the bundle deployment
   
    * - `lock`
      - Map
-     - The deployment lock attributes. See [_](#bundle.deployment.lock).
+     - The deployment lock attributes. See [_](#bundledeploymentlock).
   
   
 ### bundle.deployment.lock
@@ -180,7 +180,7 @@ The deployment lock attributes.
   
 **`Type: Map`**
   
-The Git version control details that are associated with your bundle.
+The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git).
   
   
   
@@ -217,11 +217,11 @@ Defines attributes for experimental features.
   
    * - `pydabs`
      - Map
-     - The PyDABs configuration. See [_](#experimental.pydabs).
+     - The PyDABs configuration. See [_](#experimentalpydabs).
   
    * - `python`
      - Map
-     - Configures loading of Python code defined with 'databricks-bundles' package. See [_](#experimental.python).
+     - Configures loading of Python code defined with 'databricks-bundles' package. See [_](#experimentalpython).
   
    * - `python_wheel_wrapper`
      - Boolean
@@ -530,11 +530,11 @@ targets:
   
    * - `artifacts`
      - Map
-     - The artifacts to include in the target deployment. See [_](#targets.<name>.artifacts).
+     - The artifacts to include in the target deployment. See [_](#targetsnameartifacts).
   
    * - `bundle`
      - Map
-     - The bundle attributes when deploying to this target. See [_](#targets.<name>.bundle).
+     - The bundle attributes when deploying to this target. See [_](#targetsnamebundle).
   
    * - `cluster_id`
      - String
@@ -550,7 +550,7 @@ targets:
   
    * - `git`
      - Map
-     - The Git version control settings for the target. See [_](#targets.<name>.git).
+     - The Git version control settings for the target. See [_](#targetsnamegit).
   
    * - `mode`
      - String
@@ -558,34 +558,34 @@ targets:
   
    * - `permissions`
      - Sequence
-     - The permissions for deploying and running the bundle in the target. See [_](#targets.<name>.permissions).
+     - The permissions for deploying and running the bundle in the target. See [_](#targetsnamepermissions).
   
    * - `presets`
      - Map
-     - The deployment presets for the target. See [_](#targets.<name>.presets).
+     - The deployment presets for the target. See [_](#targetsnamepresets).
   
    * - `resources`
      - Map
-     - The resource definitions for the target. See [_](#targets.<name>.resources).
+     - The resource definitions for the target. See [_](#targetsnameresources).
   
    * - `run_as`
      - Map
-     - The identity to use to run the bundle, see [_](/dev-tools/bundles/run-as.md). See [_](#targets.<name>.run_as).
+     - The identity to use to run the bundle, see [_](/dev-tools/bundles/run-as.md). See [_](#targetsnamerun_as).
   
    * - `sync`
      - Map
-     - The local paths to sync to the target workspace when a bundle is run or deployed. See [_](#targets.<name>.sync).
+     - The local paths to sync to the target workspace when a bundle is run or deployed. See [_](#targetsnamesync).
   
    * - `variables`
      - Map
-     - The custom variable definitions for the target. See [_](#targets.<name>.variables).
+     - The custom variable definitions for the target. See [_](#targetsnamevariables).
   
    * - `workspace`
      - Map
-     - The Databricks workspace for the target. See [_](#targets.<name>.workspace).
+     - The Databricks workspace for the target. See [_](#targetsnameworkspace).
   
   
-### targets.<name>.artifacts
+### targets.\<name\>.artifacts
   
 **`Type: Map`**
   
@@ -615,7 +615,7 @@ artifacts:
   
    * - `files`
      - Sequence
-     - The source files for the artifact. See [_](#targets.<name>.artifacts.<name>.files).
+     - The source files for the artifact. See [_](#targetsnameartifactsnamefiles).
   
    * - `path`
      - String
@@ -626,7 +626,7 @@ artifacts:
      - Required. The type of the artifact. Valid values are `whl`.
   
   
-### targets.<name>.artifacts.<name>.files
+### targets.\<name\>.artifacts.\<name\>.files
   
 **`Type: Sequence`**
   
@@ -646,7 +646,7 @@ The source files for the artifact.
      - Required. The path of the files used to build the artifact.
   
   
-### targets.<name>.bundle
+### targets.\<name\>.bundle
   
 **`Type: Map`**
   
@@ -675,11 +675,11 @@ The bundle attributes when deploying to this target.
   
    * - `deployment`
      - Map
-     - The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). See [_](#targets.<name>.bundle.deployment).
+     - The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). See [_](#targetsnamebundledeployment).
   
    * - `git`
      - Map
-     - The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). See [_](#targets.<name>.bundle.git).
+     - The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). See [_](#targetsnamebundlegit).
   
    * - `name`
      - String
@@ -690,11 +690,11 @@ The bundle attributes when deploying to this target.
      - Reserved. A Universally Unique Identifier (UUID) for the bundle that uniquely identifies the bundle in internal Databricks systems. This is generated when a bundle project is initialized using a Databricks template (using the `databricks bundle init` command).
   
   
-### targets.<name>.bundle.deployment
+### targets.\<name\>.bundle.deployment
   
 **`Type: Map`**
   
-The definition of the bundle deployment
+The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md).
   
   
   
@@ -711,10 +711,10 @@ The definition of the bundle deployment
   
    * - `lock`
      - Map
-     - The deployment lock attributes. See [_](#targets.<name>.bundle.deployment.lock).
+     - The deployment lock attributes. See [_](#targetsnamebundledeploymentlock).
   
   
-### targets.<name>.bundle.deployment.lock
+### targets.\<name\>.bundle.deployment.lock
   
 **`Type: Map`**
   
@@ -738,11 +738,11 @@ The deployment lock attributes.
      - Whether to force this lock if it is enabled.
   
   
-### targets.<name>.bundle.git
+### targets.\<name\>.bundle.git
   
 **`Type: Map`**
   
-The Git version control details that are associated with your bundle.
+The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git).
   
   
   
@@ -762,7 +762,7 @@ The Git version control details that are associated with your bundle.
      - The origin URL of the repository. See [_](/dev-tools/bundles/settings.md#git).
   
   
-### targets.<name>.git
+### targets.\<name\>.git
   
 **`Type: Map`**
   
@@ -786,7 +786,7 @@ The Git version control settings for the target.
      - The origin URL of the repository. See [_](/dev-tools/bundles/settings.md#git).
   
   
-### targets.<name>.permissions
+### targets.\<name\>.permissions
   
 **`Type: Sequence`**
   
@@ -818,7 +818,7 @@ The permissions for deploying and running the bundle in the target.
      - The name of the user that has the permission set in level.
   
   
-### targets.<name>.presets
+### targets.\<name\>.presets
   
 **`Type: Map`**
   
@@ -858,7 +858,7 @@ The deployment presets for the target.
      - A pause status to apply to all job triggers and schedules. Valid values are PAUSED or UNPAUSED.
   
   
-### targets.<name>.resources
+### targets.\<name\>.resources
   
 **`Type: Map`**
   
@@ -922,11 +922,11 @@ The resource definitions for the target.
      - The volume definitions for the bundle, where each key is the name of the volume. See [_](/dev-tools/bundles/resources.md#volumes)
   
   
-### targets.<name>.run_as
+### targets.\<name\>.run_as
   
 **`Type: Map`**
   
-The identity to use to run the bundle.
+The identity to use to run the bundle, see [_](/dev-tools/bundles/run-as.md).
   
   
   
@@ -946,7 +946,7 @@ The identity to use to run the bundle.
      - The email of an active workspace user. Non-admin users can only set this field to their own email.
   
   
-### targets.<name>.sync
+### targets.\<name\>.sync
   
 **`Type: Map`**
   
@@ -974,7 +974,7 @@ The local paths to sync to the target workspace when a bundle is run or deployed
      - The local folder paths, which can be outside the bundle root, to synchronize to the workspace when the bundle is deployed.
   
   
-### targets.<name>.variables
+### targets.\<name\>.variables
   
 **`Type: Map`**
   
@@ -1004,14 +1004,14 @@ variables:
   
    * - `lookup`
      - Map
-     - The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID. See [_](#targets.<name>.variables.<name>.lookup).
+     - The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID. See [_](#targetsnamevariablesnamelookup).
   
    * - `type`
      - String
      - The type of the variable.
   
   
-### targets.<name>.variables.<name>.lookup
+### targets.\<name\>.variables.\<name\>.lookup
   
 **`Type: Map`**
   
@@ -1075,7 +1075,7 @@ The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, m
      - 
   
   
-### targets.<name>.workspace
+### targets.\<name\>.workspace
   
 **`Type: Map`**
   
@@ -1185,18 +1185,18 @@ variables:
   
    * - `lookup`
      - Map
-     - The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID. See [_](#variables.<name>.lookup).
+     - The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID. See [_](#variablesnamelookup).
   
    * - `type`
      - String
      - The type of the variable.
   
   
-### variables.<name>.lookup
+### variables.\<name\>.lookup
   
 **`Type: Map`**
   
-The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID.
+The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID.
   
   
   
diff --git a/bundle/docsgen/output/resources.md b/bundle/docsgen/output/resources.md
index df7578c73..e1bbc9672 100644
--- a/bundle/docsgen/output/resources.md
+++ b/bundle/docsgen/output/resources.md
@@ -93,15 +93,15 @@ apps:
   
    * - `active_deployment`
      - Map
-     - See [_](#apps.<name>.active_deployment).
+     - See [_](#appsnameactive_deployment).
   
    * - `app_status`
      - Map
-     - See [_](#apps.<name>.app_status).
+     - See [_](#appsnameapp_status).
   
    * - `compute_status`
      - Map
-     - See [_](#apps.<name>.compute_status).
+     - See [_](#appsnamecompute_status).
   
    * - `config`
      - Map
@@ -129,15 +129,15 @@ apps:
   
    * - `pending_deployment`
      - Map
-     - See [_](#apps.<name>.pending_deployment).
+     - See [_](#appsnamepending_deployment).
   
    * - `permissions`
      - Sequence
-     - See [_](#apps.<name>.permissions).
+     - See [_](#appsnamepermissions).
   
    * - `resources`
      - Sequence
-     - See [_](#apps.<name>.resources).
+     - See [_](#appsnameresources).
   
    * - `service_principal_client_id`
      - String
@@ -168,7 +168,7 @@ apps:
      - 
   
   
-### apps.<name>.active_deployment
+### apps.\<name\>.active_deployment
   
 **`Type: Map`**
   
@@ -193,7 +193,7 @@ apps:
   
    * - `deployment_artifacts`
      - Map
-     - See [_](#apps.<name>.active_deployment.deployment_artifacts).
+     - See [_](#appsnameactive_deploymentdeployment_artifacts).
   
    * - `deployment_id`
      - String
@@ -209,14 +209,14 @@ apps:
   
    * - `status`
      - Map
-     - See [_](#apps.<name>.active_deployment.status).
+     - See [_](#appsnameactive_deploymentstatus).
   
    * - `update_time`
      - String
      - 
   
   
-### apps.<name>.active_deployment.deployment_artifacts
+### apps.\<name\>.active_deployment.deployment_artifacts
   
 **`Type: Map`**
   
@@ -236,7 +236,7 @@ apps:
      - 
   
   
-### apps.<name>.active_deployment.status
+### apps.\<name\>.active_deployment.status
   
 **`Type: Map`**
   
@@ -260,7 +260,7 @@ apps:
      - 
   
   
-### apps.<name>.app_status
+### apps.\<name\>.app_status
   
 **`Type: Map`**
   
@@ -284,7 +284,7 @@ apps:
      - 
   
   
-### apps.<name>.compute_status
+### apps.\<name\>.compute_status
   
 **`Type: Map`**
   
@@ -308,7 +308,7 @@ apps:
      - State of the app compute.
   
   
-### apps.<name>.pending_deployment
+### apps.\<name\>.pending_deployment
   
 **`Type: Map`**
   
@@ -333,7 +333,7 @@ apps:
   
    * - `deployment_artifacts`
      - Map
-     - See [_](#apps.<name>.pending_deployment.deployment_artifacts).
+     - See [_](#appsnamepending_deploymentdeployment_artifacts).
   
    * - `deployment_id`
      - String
@@ -349,14 +349,14 @@ apps:
   
    * - `status`
      - Map
-     - See [_](#apps.<name>.pending_deployment.status).
+     - See [_](#appsnamepending_deploymentstatus).
   
    * - `update_time`
      - String
      - 
   
   
-### apps.<name>.pending_deployment.deployment_artifacts
+### apps.\<name\>.pending_deployment.deployment_artifacts
   
 **`Type: Map`**
   
@@ -376,7 +376,7 @@ apps:
      - 
   
   
-### apps.<name>.pending_deployment.status
+### apps.\<name\>.pending_deployment.status
   
 **`Type: Map`**
   
@@ -400,7 +400,7 @@ apps:
      - 
   
   
-### apps.<name>.permissions
+### apps.\<name\>.permissions
   
 **`Type: Sequence`**
   
@@ -432,7 +432,7 @@ apps:
      - The name of the user that has the permission set in level.
   
   
-### apps.<name>.resources
+### apps.\<name\>.resources
   
 **`Type: Sequence`**
   
@@ -453,7 +453,7 @@ apps:
   
    * - `job`
      - Map
-     - See [_](#apps.<name>.resources.job).
+     - See [_](#appsnameresourcesjob).
   
    * - `name`
      - String
@@ -461,18 +461,18 @@ apps:
   
    * - `secret`
      - Map
-     - See [_](#apps.<name>.resources.secret).
+     - See [_](#appsnameresourcessecret).
   
    * - `serving_endpoint`
      - Map
-     - See [_](#apps.<name>.resources.serving_endpoint).
+     - See [_](#appsnameresourcesserving_endpoint).
   
    * - `sql_warehouse`
      - Map
-     - See [_](#apps.<name>.resources.sql_warehouse).
+     - See [_](#appsnameresourcessql_warehouse).
   
   
-### apps.<name>.resources.job
+### apps.\<name\>.resources.job
   
 **`Type: Map`**
   
@@ -496,7 +496,7 @@ apps:
      - 
   
   
-### apps.<name>.resources.secret
+### apps.\<name\>.resources.secret
   
 **`Type: Map`**
   
@@ -524,7 +524,7 @@ apps:
      - 
   
   
-### apps.<name>.resources.serving_endpoint
+### apps.\<name\>.resources.serving_endpoint
   
 **`Type: Map`**
   
@@ -548,7 +548,7 @@ apps:
      - 
   
   
-### apps.<name>.resources.sql_warehouse
+### apps.\<name\>.resources.sql_warehouse
   
 **`Type: Map`**
   
@@ -598,7 +598,7 @@ clusters:
   
    * - `autoscale`
      - Map
-     - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#clusters.<name>.autoscale).
+     - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#clustersnameautoscale).
   
    * - `autotermination_minutes`
      - Integer
@@ -606,15 +606,15 @@ clusters:
   
    * - `aws_attributes`
      - Map
-     - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#clusters.<name>.aws_attributes).
+     - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#clustersnameaws_attributes).
   
    * - `azure_attributes`
      - Map
-     - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#clusters.<name>.azure_attributes).
+     - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#clustersnameazure_attributes).
   
    * - `cluster_log_conf`
      - Map
-     - The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. See [_](#clusters.<name>.cluster_log_conf).
+     - The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. See [_](#clustersnamecluster_log_conf).
   
    * - `cluster_name`
      - String
@@ -630,7 +630,7 @@ clusters:
   
    * - `docker_image`
      - Map
-     - See [_](#clusters.<name>.docker_image).
+     - See [_](#clustersnamedocker_image).
   
    * - `driver_instance_pool_id`
      - String
@@ -650,11 +650,11 @@ clusters:
   
    * - `gcp_attributes`
      - Map
-     - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#clusters.<name>.gcp_attributes).
+     - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#clustersnamegcp_attributes).
   
    * - `init_scripts`
      - Sequence
-     - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `<destination>/<cluster-ID>/init_scripts`. See [_](#clusters.<name>.init_scripts).
+     - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `<destination>/<cluster-ID>/init_scripts`. See [_](#clustersnameinit_scripts).
   
    * - `instance_pool_id`
      - String
@@ -678,7 +678,7 @@ clusters:
   
    * - `permissions`
      - Sequence
-     - See [_](#clusters.<name>.permissions).
+     - See [_](#clustersnamepermissions).
   
    * - `policy_id`
      - String
@@ -714,7 +714,7 @@ clusters:
   
    * - `workload_type`
      - Map
-     - See [_](#clusters.<name>.workload_type).
+     - See [_](#clustersnameworkload_type).
   
   
 **Example**
@@ -745,7 +745,7 @@ resources:
             notebook_path: "./src/my_notebook.py"
 ```
   
-### clusters.<name>.autoscale
+### clusters.\<name\>.autoscale
   
 **`Type: Map`**
   
@@ -770,7 +770,7 @@ Note: autoscaling works best with DB runtime versions 3.0 or later.
      - The minimum number of workers to which the cluster can scale down when underutilized. It is also the initial number of workers the cluster will have after creation.
   
   
-### clusters.<name>.aws_attributes
+### clusters.\<name\>.aws_attributes
   
 **`Type: Map`**
   
@@ -827,7 +827,7 @@ If not specified at cluster creation, a set of default values will be used.
      - Identifier for the availability zone/datacenter in which the cluster resides. This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. This is an optional field at cluster creation, and if not specified, a default zone will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. The list of available zones as well as the default value can be found by using the `List Zones` method.
   
   
-### clusters.<name>.azure_attributes
+### clusters.\<name\>.azure_attributes
   
 **`Type: Map`**
   
@@ -853,14 +853,14 @@ If not specified at cluster creation, a set of default values will be used.
   
    * - `log_analytics_info`
      - Map
-     - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#clusters.<name>.azure_attributes.log_analytics_info).
+     - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#clustersnameazure_attributeslog_analytics_info).
   
    * - `spot_bid_max_price`
      - Any
      - The max bid price to be used for Azure spot instances. The Max price for the bid cannot be higher than the on-demand price of the instance. If not specified, the default value is -1, which specifies that the instance cannot be evicted on the basis of price, and only on the basis of availability. Further, the value should > 0 or -1.
   
   
-### clusters.<name>.azure_attributes.log_analytics_info
+### clusters.\<name\>.azure_attributes.log_analytics_info
   
 **`Type: Map`**
   
@@ -884,7 +884,7 @@ Defines values necessary to configure and run Azure Log Analytics agent
      - <needs content added>
   
   
-### clusters.<name>.cluster_log_conf
+### clusters.\<name\>.cluster_log_conf
   
 **`Type: Map`**
   
@@ -905,14 +905,14 @@ the destination of executor logs is `$destination/$clusterId/executor`.
   
    * - `dbfs`
      - Map
-     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#clusters.<name>.cluster_log_conf.dbfs).
+     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#clustersnamecluster_log_confdbfs).
   
    * - `s3`
      - Map
-     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#clusters.<name>.cluster_log_conf.s3).
+     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#clustersnamecluster_log_confs3).
   
   
-### clusters.<name>.cluster_log_conf.dbfs
+### clusters.\<name\>.cluster_log_conf.dbfs
   
 **`Type: Map`**
   
@@ -933,7 +933,7 @@ destination needs to be provided. e.g.
      - dbfs destination, e.g. `dbfs:/my/path`
   
   
-### clusters.<name>.cluster_log_conf.s3
+### clusters.\<name\>.cluster_log_conf.s3
   
 **`Type: Map`**
   
@@ -980,7 +980,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in
      - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
   
   
-### clusters.<name>.docker_image
+### clusters.\<name\>.docker_image
   
 **`Type: Map`**
   
@@ -997,14 +997,14 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in
   
    * - `basic_auth`
      - Map
-     - See [_](#clusters.<name>.docker_image.basic_auth).
+     - See [_](#clustersnamedocker_imagebasic_auth).
   
    * - `url`
      - String
      - URL of the docker image.
   
   
-### clusters.<name>.docker_image.basic_auth
+### clusters.\<name\>.docker_image.basic_auth
   
 **`Type: Map`**
   
@@ -1028,7 +1028,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in
      - Name of the user
   
   
-### clusters.<name>.gcp_attributes
+### clusters.\<name\>.gcp_attributes
   
 **`Type: Map`**
   
@@ -1069,7 +1069,7 @@ If not specified at cluster creation, a set of default values will be used.
      - Identifier for the availability zone in which the cluster resides. This can be one of the following: - "HA" => High availability, spread nodes across availability zones for a Databricks deployment region [default] - "AUTO" => Databricks picks an availability zone to schedule the cluster on. - A GCP availability zone => Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones.
   
   
-### clusters.<name>.init_scripts
+### clusters.\<name\>.init_scripts
   
 **`Type: Sequence`**
   
@@ -1086,34 +1086,34 @@ The configuration for storing init scripts. Any number of destinations can be sp
   
    * - `abfss`
      - Map
-     - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>" } }. See [_](#clusters.<name>.init_scripts.abfss).
+     - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>" } }. See [_](#clustersnameinit_scriptsabfss).
   
    * - `dbfs`
      - Map
-     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#clusters.<name>.init_scripts.dbfs).
+     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#clustersnameinit_scriptsdbfs).
   
    * - `file`
      - Map
-     - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#clusters.<name>.init_scripts.file).
+     - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#clustersnameinit_scriptsfile).
   
    * - `gcs`
      - Map
-     - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#clusters.<name>.init_scripts.gcs).
+     - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#clustersnameinit_scriptsgcs).
   
    * - `s3`
      - Map
-     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#clusters.<name>.init_scripts.s3).
+     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#clustersnameinit_scriptss3).
   
    * - `volumes`
      - Map
-     - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#clusters.<name>.init_scripts.volumes).
+     - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#clustersnameinit_scriptsvolumes).
   
    * - `workspace`
      - Map
-     - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#clusters.<name>.init_scripts.workspace).
+     - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#clustersnameinit_scriptsworkspace).
   
   
-### clusters.<name>.init_scripts.abfss
+### clusters.\<name\>.init_scripts.abfss
   
 **`Type: Map`**
   
@@ -1134,7 +1134,7 @@ destination needs to be provided. e.g.
      - abfss destination, e.g. `abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>`.
   
   
-### clusters.<name>.init_scripts.dbfs
+### clusters.\<name\>.init_scripts.dbfs
   
 **`Type: Map`**
   
@@ -1155,7 +1155,7 @@ destination needs to be provided. e.g.
      - dbfs destination, e.g. `dbfs:/my/path`
   
   
-### clusters.<name>.init_scripts.file
+### clusters.\<name\>.init_scripts.file
   
 **`Type: Map`**
   
@@ -1176,7 +1176,7 @@ destination needs to be provided. e.g.
      - local file destination, e.g. `file:/my/local/file.sh`
   
   
-### clusters.<name>.init_scripts.gcs
+### clusters.\<name\>.init_scripts.gcs
   
 **`Type: Map`**
   
@@ -1197,7 +1197,7 @@ destination needs to be provided. e.g.
      - GCS destination/URI, e.g. `gs://my-bucket/some-prefix`
   
   
-### clusters.<name>.init_scripts.s3
+### clusters.\<name\>.init_scripts.s3
   
 **`Type: Map`**
   
@@ -1244,7 +1244,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in
      - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
   
   
-### clusters.<name>.init_scripts.volumes
+### clusters.\<name\>.init_scripts.volumes
   
 **`Type: Map`**
   
@@ -1265,7 +1265,7 @@ destination needs to be provided. e.g.
      - Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`
   
   
-### clusters.<name>.init_scripts.workspace
+### clusters.\<name\>.init_scripts.workspace
   
 **`Type: Map`**
   
@@ -1286,7 +1286,7 @@ destination needs to be provided. e.g.
      - workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`
   
   
-### clusters.<name>.permissions
+### clusters.\<name\>.permissions
   
 **`Type: Sequence`**
   
@@ -1318,7 +1318,7 @@ destination needs to be provided. e.g.
      - The name of the user that has the permission set in level.
   
   
-### clusters.<name>.workload_type
+### clusters.\<name\>.workload_type
   
 **`Type: Map`**
   
@@ -1335,10 +1335,10 @@ destination needs to be provided. e.g.
   
    * - `clients`
      - Map
-     -  defined what type of clients can use the cluster. E.g. Notebooks, Jobs. See [_](#clusters.<name>.workload_type.clients).
+     -  defined what type of clients can use the cluster. E.g. Notebooks, Jobs. See [_](#clustersnameworkload_typeclients).
   
   
-### clusters.<name>.workload_type.clients
+### clusters.\<name\>.workload_type.clients
   
 **`Type: Map`**
   
@@ -1420,7 +1420,7 @@ dashboards:
   
    * - `permissions`
      - Sequence
-     - See [_](#dashboards.<name>.permissions).
+     - See [_](#dashboardsnamepermissions).
   
    * - `serialized_dashboard`
      - Any
@@ -1451,7 +1451,7 @@ If you use the UI to modify the dashboard, modifications made through the UI are
 
 In addition, if you attempt to deploy a bundle that contains a dashboard JSON file that is different than the one in the remote workspace, an error will occur. To force the deploy and overwrite the dashboard in the remote workspace with the local one, use the `--force` option. See [_](/dev-tools/cli/bundle-commands.md#deploy).
   
-### dashboards.<name>.permissions
+### dashboards.\<name\>.permissions
   
 **`Type: Sequence`**
   
@@ -1529,11 +1529,11 @@ experiments:
   
    * - `permissions`
      - Sequence
-     - See [_](#experiments.<name>.permissions).
+     - See [_](#experimentsnamepermissions).
   
    * - `tags`
      - Sequence
-     - Tags: Additional metadata key-value pairs. See [_](#experiments.<name>.tags).
+     - Tags: Additional metadata key-value pairs. See [_](#experimentsnametags).
   
   
 **Example**
@@ -1551,7 +1551,7 @@ resources:
       description: MLflow experiment used to track runs
 ```
   
-### experiments.<name>.permissions
+### experiments.\<name\>.permissions
   
 **`Type: Sequence`**
   
@@ -1583,7 +1583,7 @@ resources:
      - The name of the user that has the permission set in level.
   
   
-### experiments.<name>.tags
+### experiments.\<name\>.tags
   
 **`Type: Sequence`**
   
@@ -1633,11 +1633,11 @@ jobs:
   
    * - `continuous`
      - Map
-     - An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used. See [_](#jobs.<name>.continuous).
+     - An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used. See [_](#jobsnamecontinuous).
   
    * - `deployment`
      - Map
-     - Deployment information for jobs managed by external sources. See [_](#jobs.<name>.deployment).
+     - Deployment information for jobs managed by external sources. See [_](#jobsnamedeployment).
   
    * - `description`
      - String
@@ -1649,11 +1649,11 @@ jobs:
   
    * - `email_notifications`
      - Map
-     - An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted. See [_](#jobs.<name>.email_notifications).
+     - An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted. See [_](#jobsnameemail_notifications).
   
    * - `environments`
      - Sequence
-     - A list of task execution environment specifications that can be referenced by serverless tasks of this job. An environment is required to be present for serverless tasks. For serverless notebook tasks, the environment is accessible in the notebook environment panel. For other serverless tasks, the task environment is required to be specified using environment_key in the task settings. See [_](#jobs.<name>.environments).
+     - A list of task execution environment specifications that can be referenced by serverless tasks of this job. An environment is required to be present for serverless tasks. For serverless notebook tasks, the environment is accessible in the notebook environment panel. For other serverless tasks, the task environment is required to be specified using environment_key in the task settings. See [_](#jobsnameenvironments).
   
    * - `format`
      - String
@@ -1661,15 +1661,15 @@ jobs:
   
    * - `git_source`
      - Map
-     - An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.  If `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task.  Note: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job. See [_](#jobs.<name>.git_source).
+     - An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.  If `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task.  Note: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job. See [_](#jobsnamegit_source).
   
    * - `health`
      - Map
-     - An optional set of health rules that can be defined for this job. See [_](#jobs.<name>.health).
+     - An optional set of health rules that can be defined for this job. See [_](#jobsnamehealth).
   
    * - `job_clusters`
      - Sequence
-     - A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings. If more than 100 job clusters are available, you can paginate through them using :method:jobs/get. See [_](#jobs.<name>.job_clusters).
+     - A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings. If more than 100 job clusters are available, you can paginate through them using :method:jobs/get. See [_](#jobsnamejob_clusters).
   
    * - `max_concurrent_runs`
      - Integer
@@ -1681,27 +1681,31 @@ jobs:
   
    * - `notification_settings`
      - Map
-     - Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this job. See [_](#jobs.<name>.notification_settings).
+     - Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this job. See [_](#jobsnamenotification_settings).
   
    * - `parameters`
      - Sequence
-     - Job-level parameter definitions. See [_](#jobs.<name>.parameters).
+     - Job-level parameter definitions. See [_](#jobsnameparameters).
+  
+   * - `performance_target`
+     - String
+     - PerformanceTarget defines how performant or cost efficient the execution of run on serverless should be.
   
    * - `permissions`
      - Sequence
-     - See [_](#jobs.<name>.permissions).
+     - See [_](#jobsnamepermissions).
   
    * - `queue`
      - Map
-     - The queue settings of the job. See [_](#jobs.<name>.queue).
+     - The queue settings of the job. See [_](#jobsnamequeue).
   
    * - `run_as`
      - Map
-     - Write-only setting. Specifies the user or service principal that the job runs as. If not specified, the job runs as the user who created the job.  Either `user_name` or `service_principal_name` should be specified. If not, an error is thrown. See [_](#jobs.<name>.run_as).
+     - Write-only setting. Specifies the user or service principal that the job runs as. If not specified, the job runs as the user who created the job.  Either `user_name` or `service_principal_name` should be specified. If not, an error is thrown. See [_](#jobsnamerun_as).
   
    * - `schedule`
      - Map
-     - An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`. See [_](#jobs.<name>.schedule).
+     - An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`. See [_](#jobsnameschedule).
   
    * - `tags`
      - Map
@@ -1709,7 +1713,7 @@ jobs:
   
    * - `tasks`
      - Sequence
-     - A list of task specifications to be executed by this job. If more than 100 tasks are available, you can paginate through them using :method:jobs/get. Use the `next_page_token` field at the object root to determine if more results are available. See [_](#jobs.<name>.tasks).
+     - A list of task specifications to be executed by this job. If more than 100 tasks are available, you can paginate through them using :method:jobs/get. Use the `next_page_token` field at the object root to determine if more results are available. See [_](#jobsnametasks).
   
    * - `timeout_seconds`
      - Integer
@@ -1717,11 +1721,11 @@ jobs:
   
    * - `trigger`
      - Map
-     - A configuration to trigger a run when certain conditions are met. The default behavior is that the job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`. See [_](#jobs.<name>.trigger).
+     - A configuration to trigger a run when certain conditions are met. The default behavior is that the job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`. See [_](#jobsnametrigger).
   
    * - `webhook_notifications`
      - Map
-     - A collection of system notification IDs to notify when runs of this job begin or complete. See [_](#jobs.<name>.webhook_notifications).
+     - A collection of system notification IDs to notify when runs of this job begin or complete. See [_](#jobsnamewebhook_notifications).
   
   
 **Example**
@@ -1741,7 +1745,7 @@ resources:
 
 For information about defining job tasks and overriding job settings, see [_](/dev-tools/bundles/job-task-types.md), [_](/dev-tools/bundles/job-task-override.md), and [_](/dev-tools/bundles/cluster-override.md).
   
-### jobs.<name>.continuous
+### jobs.\<name\>.continuous
   
 **`Type: Map`**
   
@@ -1761,7 +1765,7 @@ An optional continuous property for this job. The continuous property will ensur
      - Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED.
   
   
-### jobs.<name>.deployment
+### jobs.\<name\>.deployment
   
 **`Type: Map`**
   
@@ -1785,7 +1789,7 @@ Deployment information for jobs managed by external sources.
      - Path of the file that contains deployment metadata.
   
   
-### jobs.<name>.email_notifications
+### jobs.\<name\>.email_notifications
   
 **`Type: Map`**
   
@@ -1825,7 +1829,7 @@ An optional set of email addresses that is notified when runs of this job begin
      - A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.
   
   
-### jobs.<name>.environments
+### jobs.\<name\>.environments
   
 **`Type: Sequence`**
   
@@ -1849,10 +1853,10 @@ For other serverless tasks, the task environment is required to be specified usi
   
    * - `spec`
      - Map
-     - The environment entity used to preserve serverless environment side panel and jobs' environment for non-notebook task. In this minimal environment spec, only pip dependencies are supported. See [_](#jobs.<name>.environments.spec).
+     - The environment entity used to preserve serverless environment side panel and jobs' environment for non-notebook task. In this minimal environment spec, only pip dependencies are supported. See [_](#jobsnameenvironmentsspec).
   
   
-### jobs.<name>.environments.spec
+### jobs.\<name\>.environments.spec
   
 **`Type: Map`**
   
@@ -1877,7 +1881,7 @@ In this minimal environment spec, only pip dependencies are supported.
      - List of pip dependencies, as supported by the version of pip in this environment. Each dependency is a pip requirement file line https://pip.pypa.io/en/stable/reference/requirements-file-format/ Allowed dependency could be <requirement specifier>, <archive url/path>, <local project path>(WSFS or Volumes in Databricks), <vcs project url> E.g. dependencies: ["foo==0.0.1", "-r /Workspace/test/requirements.txt"]
   
   
-### jobs.<name>.git_source
+### jobs.\<name\>.git_source
   
 **`Type: Map`**
   
@@ -1910,7 +1914,7 @@ Note: dbt and SQL File tasks support only version-controlled sources. If dbt or
   
    * - `git_snapshot`
      - Map
-     - Read-only state of the remote repository at the time the job was run. This field is only included on job runs. See [_](#jobs.<name>.git_source.git_snapshot).
+     - Read-only state of the remote repository at the time the job was run. This field is only included on job runs. See [_](#jobsnamegit_sourcegit_snapshot).
   
    * - `git_tag`
      - String
@@ -1922,10 +1926,10 @@ Note: dbt and SQL File tasks support only version-controlled sources. If dbt or
   
    * - `job_source`
      - Map
-     - The source of the job specification in the remote repository when the job is source controlled. See [_](#jobs.<name>.git_source.job_source).
+     - The source of the job specification in the remote repository when the job is source controlled. See [_](#jobsnamegit_sourcejob_source).
   
   
-### jobs.<name>.git_source.git_snapshot
+### jobs.\<name\>.git_source.git_snapshot
   
 **`Type: Map`**
   
@@ -1945,7 +1949,7 @@ Read-only state of the remote repository at the time the job was run. This field
      - Commit that was used to execute the run. If git_branch was specified, this points to the HEAD of the branch at the time of the run; if git_tag was specified, this points to the commit the tag points to.
   
   
-### jobs.<name>.git_source.job_source
+### jobs.\<name\>.git_source.job_source
   
 **`Type: Map`**
   
@@ -1973,7 +1977,7 @@ The source of the job specification in the remote repository when the job is sou
      - Path of the job YAML file that contains the job specification.
   
   
-### jobs.<name>.health
+### jobs.\<name\>.health
   
 **`Type: Map`**
   
@@ -1990,10 +1994,10 @@ An optional set of health rules that can be defined for this job.
   
    * - `rules`
      - Sequence
-     - See [_](#jobs.<name>.health.rules).
+     - See [_](#jobsnamehealthrules).
   
   
-### jobs.<name>.health.rules
+### jobs.\<name\>.health.rules
   
 **`Type: Sequence`**
   
@@ -2021,7 +2025,7 @@ An optional set of health rules that can be defined for this job.
      - Specifies the threshold value that the health metric should obey to satisfy the health rule.
   
   
-### jobs.<name>.job_clusters
+### jobs.\<name\>.job_clusters
   
 **`Type: Sequence`**
   
@@ -2043,10 +2047,10 @@ If more than 100 job clusters are available, you can paginate through them using
   
    * - `new_cluster`
      - Map
-     - If new_cluster, a description of a cluster that is created for each task. See [_](#jobs.<name>.job_clusters.new_cluster).
+     - If new_cluster, a description of a cluster that is created for each task. See [_](#jobsnamejob_clustersnew_cluster).
   
   
-### jobs.<name>.job_clusters.new_cluster
+### jobs.\<name\>.job_clusters.new_cluster
   
 **`Type: Map`**
   
@@ -2067,7 +2071,7 @@ If new_cluster, a description of a cluster that is created for each task.
   
    * - `autoscale`
      - Map
-     - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#jobs.<name>.job_clusters.new_cluster.autoscale).
+     - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#jobsnamejob_clustersnew_clusterautoscale).
   
    * - `autotermination_minutes`
      - Integer
@@ -2075,15 +2079,15 @@ If new_cluster, a description of a cluster that is created for each task.
   
    * - `aws_attributes`
      - Map
-     - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#jobs.<name>.job_clusters.new_cluster.aws_attributes).
+     - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#jobsnamejob_clustersnew_clusteraws_attributes).
   
    * - `azure_attributes`
      - Map
-     - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#jobs.<name>.job_clusters.new_cluster.azure_attributes).
+     - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#jobsnamejob_clustersnew_clusterazure_attributes).
   
    * - `cluster_log_conf`
      - Map
-     - The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. See [_](#jobs.<name>.job_clusters.new_cluster.cluster_log_conf).
+     - The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. See [_](#jobsnamejob_clustersnew_clustercluster_log_conf).
   
    * - `cluster_name`
      - String
@@ -2099,7 +2103,7 @@ If new_cluster, a description of a cluster that is created for each task.
   
    * - `docker_image`
      - Map
-     - See [_](#jobs.<name>.job_clusters.new_cluster.docker_image).
+     - See [_](#jobsnamejob_clustersnew_clusterdocker_image).
   
    * - `driver_instance_pool_id`
      - String
@@ -2119,11 +2123,11 @@ If new_cluster, a description of a cluster that is created for each task.
   
    * - `gcp_attributes`
      - Map
-     - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#jobs.<name>.job_clusters.new_cluster.gcp_attributes).
+     - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#jobsnamejob_clustersnew_clustergcp_attributes).
   
    * - `init_scripts`
      - Sequence
-     - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `<destination>/<cluster-ID>/init_scripts`. See [_](#jobs.<name>.job_clusters.new_cluster.init_scripts).
+     - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `<destination>/<cluster-ID>/init_scripts`. See [_](#jobsnamejob_clustersnew_clusterinit_scripts).
   
    * - `instance_pool_id`
      - String
@@ -2179,10 +2183,10 @@ If new_cluster, a description of a cluster that is created for each task.
   
    * - `workload_type`
      - Map
-     - See [_](#jobs.<name>.job_clusters.new_cluster.workload_type).
+     - See [_](#jobsnamejob_clustersnew_clusterworkload_type).
   
   
-### jobs.<name>.job_clusters.new_cluster.autoscale
+### jobs.\<name\>.job_clusters.new_cluster.autoscale
   
 **`Type: Map`**
   
@@ -2207,7 +2211,7 @@ Note: autoscaling works best with DB runtime versions 3.0 or later.
      - The minimum number of workers to which the cluster can scale down when underutilized. It is also the initial number of workers the cluster will have after creation.
   
   
-### jobs.<name>.job_clusters.new_cluster.aws_attributes
+### jobs.\<name\>.job_clusters.new_cluster.aws_attributes
   
 **`Type: Map`**
   
@@ -2264,7 +2268,7 @@ If not specified at cluster creation, a set of default values will be used.
      - Identifier for the availability zone/datacenter in which the cluster resides. This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. This is an optional field at cluster creation, and if not specified, a default zone will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. The list of available zones as well as the default value can be found by using the `List Zones` method.
   
   
-### jobs.<name>.job_clusters.new_cluster.azure_attributes
+### jobs.\<name\>.job_clusters.new_cluster.azure_attributes
   
 **`Type: Map`**
   
@@ -2290,14 +2294,14 @@ If not specified at cluster creation, a set of default values will be used.
   
    * - `log_analytics_info`
      - Map
-     - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#jobs.<name>.job_clusters.new_cluster.azure_attributes.log_analytics_info).
+     - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#jobsnamejob_clustersnew_clusterazure_attributeslog_analytics_info).
   
    * - `spot_bid_max_price`
      - Any
      - The max bid price to be used for Azure spot instances. The Max price for the bid cannot be higher than the on-demand price of the instance. If not specified, the default value is -1, which specifies that the instance cannot be evicted on the basis of price, and only on the basis of availability. Further, the value should > 0 or -1.
   
   
-### jobs.<name>.job_clusters.new_cluster.azure_attributes.log_analytics_info
+### jobs.\<name\>.job_clusters.new_cluster.azure_attributes.log_analytics_info
   
 **`Type: Map`**
   
@@ -2321,7 +2325,7 @@ Defines values necessary to configure and run Azure Log Analytics agent
      - <needs content added>
   
   
-### jobs.<name>.job_clusters.new_cluster.cluster_log_conf
+### jobs.\<name\>.job_clusters.new_cluster.cluster_log_conf
   
 **`Type: Map`**
   
@@ -2342,14 +2346,14 @@ the destination of executor logs is `$destination/$clusterId/executor`.
   
    * - `dbfs`
      - Map
-     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobs.<name>.job_clusters.new_cluster.cluster_log_conf.dbfs).
+     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobsnamejob_clustersnew_clustercluster_log_confdbfs).
   
    * - `s3`
      - Map
-     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobs.<name>.job_clusters.new_cluster.cluster_log_conf.s3).
+     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobsnamejob_clustersnew_clustercluster_log_confs3).
   
   
-### jobs.<name>.job_clusters.new_cluster.cluster_log_conf.dbfs
+### jobs.\<name\>.job_clusters.new_cluster.cluster_log_conf.dbfs
   
 **`Type: Map`**
   
@@ -2370,7 +2374,7 @@ destination needs to be provided. e.g.
      - dbfs destination, e.g. `dbfs:/my/path`
   
   
-### jobs.<name>.job_clusters.new_cluster.cluster_log_conf.s3
+### jobs.\<name\>.job_clusters.new_cluster.cluster_log_conf.s3
   
 **`Type: Map`**
   
@@ -2417,7 +2421,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in
      - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
   
   
-### jobs.<name>.job_clusters.new_cluster.docker_image
+### jobs.\<name\>.job_clusters.new_cluster.docker_image
   
 **`Type: Map`**
   
@@ -2434,14 +2438,14 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in
   
    * - `basic_auth`
      - Map
-     - See [_](#jobs.<name>.job_clusters.new_cluster.docker_image.basic_auth).
+     - See [_](#jobsnamejob_clustersnew_clusterdocker_imagebasic_auth).
   
    * - `url`
      - String
      - URL of the docker image.
   
   
-### jobs.<name>.job_clusters.new_cluster.docker_image.basic_auth
+### jobs.\<name\>.job_clusters.new_cluster.docker_image.basic_auth
   
 **`Type: Map`**
   
@@ -2465,7 +2469,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in
      - Name of the user
   
   
-### jobs.<name>.job_clusters.new_cluster.gcp_attributes
+### jobs.\<name\>.job_clusters.new_cluster.gcp_attributes
   
 **`Type: Map`**
   
@@ -2506,7 +2510,7 @@ If not specified at cluster creation, a set of default values will be used.
      - Identifier for the availability zone in which the cluster resides. This can be one of the following: - "HA" => High availability, spread nodes across availability zones for a Databricks deployment region [default] - "AUTO" => Databricks picks an availability zone to schedule the cluster on. - A GCP availability zone => Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones.
   
   
-### jobs.<name>.job_clusters.new_cluster.init_scripts
+### jobs.\<name\>.job_clusters.new_cluster.init_scripts
   
 **`Type: Sequence`**
   
@@ -2523,34 +2527,34 @@ The configuration for storing init scripts. Any number of destinations can be sp
   
    * - `abfss`
      - Map
-     - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>" } }. See [_](#jobs.<name>.job_clusters.new_cluster.init_scripts.abfss).
+     - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>" } }. See [_](#jobsnamejob_clustersnew_clusterinit_scriptsabfss).
   
    * - `dbfs`
      - Map
-     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobs.<name>.job_clusters.new_cluster.init_scripts.dbfs).
+     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobsnamejob_clustersnew_clusterinit_scriptsdbfs).
   
    * - `file`
      - Map
-     - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#jobs.<name>.job_clusters.new_cluster.init_scripts.file).
+     - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#jobsnamejob_clustersnew_clusterinit_scriptsfile).
   
    * - `gcs`
      - Map
-     - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#jobs.<name>.job_clusters.new_cluster.init_scripts.gcs).
+     - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#jobsnamejob_clustersnew_clusterinit_scriptsgcs).
   
    * - `s3`
      - Map
-     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobs.<name>.job_clusters.new_cluster.init_scripts.s3).
+     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobsnamejob_clustersnew_clusterinit_scriptss3).
   
    * - `volumes`
      - Map
-     - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#jobs.<name>.job_clusters.new_cluster.init_scripts.volumes).
+     - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#jobsnamejob_clustersnew_clusterinit_scriptsvolumes).
   
    * - `workspace`
      - Map
-     - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#jobs.<name>.job_clusters.new_cluster.init_scripts.workspace).
+     - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#jobsnamejob_clustersnew_clusterinit_scriptsworkspace).
   
   
-### jobs.<name>.job_clusters.new_cluster.init_scripts.abfss
+### jobs.\<name\>.job_clusters.new_cluster.init_scripts.abfss
   
 **`Type: Map`**
   
@@ -2571,7 +2575,7 @@ destination needs to be provided. e.g.
      - abfss destination, e.g. `abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>`.
   
   
-### jobs.<name>.job_clusters.new_cluster.init_scripts.dbfs
+### jobs.\<name\>.job_clusters.new_cluster.init_scripts.dbfs
   
 **`Type: Map`**
   
@@ -2592,7 +2596,7 @@ destination needs to be provided. e.g.
      - dbfs destination, e.g. `dbfs:/my/path`
   
   
-### jobs.<name>.job_clusters.new_cluster.init_scripts.file
+### jobs.\<name\>.job_clusters.new_cluster.init_scripts.file
   
 **`Type: Map`**
   
@@ -2613,7 +2617,7 @@ destination needs to be provided. e.g.
      - local file destination, e.g. `file:/my/local/file.sh`
   
   
-### jobs.<name>.job_clusters.new_cluster.init_scripts.gcs
+### jobs.\<name\>.job_clusters.new_cluster.init_scripts.gcs
   
 **`Type: Map`**
   
@@ -2634,7 +2638,7 @@ destination needs to be provided. e.g.
      - GCS destination/URI, e.g. `gs://my-bucket/some-prefix`
   
   
-### jobs.<name>.job_clusters.new_cluster.init_scripts.s3
+### jobs.\<name\>.job_clusters.new_cluster.init_scripts.s3
   
 **`Type: Map`**
   
@@ -2681,7 +2685,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in
      - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
   
   
-### jobs.<name>.job_clusters.new_cluster.init_scripts.volumes
+### jobs.\<name\>.job_clusters.new_cluster.init_scripts.volumes
   
 **`Type: Map`**
   
@@ -2702,7 +2706,7 @@ destination needs to be provided. e.g.
      - Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`
   
   
-### jobs.<name>.job_clusters.new_cluster.init_scripts.workspace
+### jobs.\<name\>.job_clusters.new_cluster.init_scripts.workspace
   
 **`Type: Map`**
   
@@ -2723,7 +2727,7 @@ destination needs to be provided. e.g.
      - workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`
   
   
-### jobs.<name>.job_clusters.new_cluster.workload_type
+### jobs.\<name\>.job_clusters.new_cluster.workload_type
   
 **`Type: Map`**
   
@@ -2740,10 +2744,10 @@ destination needs to be provided. e.g.
   
    * - `clients`
      - Map
-     -  defined what type of clients can use the cluster. E.g. Notebooks, Jobs. See [_](#jobs.<name>.job_clusters.new_cluster.workload_type.clients).
+     -  defined what type of clients can use the cluster. E.g. Notebooks, Jobs. See [_](#jobsnamejob_clustersnew_clusterworkload_typeclients).
   
   
-### jobs.<name>.job_clusters.new_cluster.workload_type.clients
+### jobs.\<name\>.job_clusters.new_cluster.workload_type.clients
   
 **`Type: Map`**
   
@@ -2767,7 +2771,7 @@ destination needs to be provided. e.g.
      - With notebooks set, this cluster can be used for notebooks
   
   
-### jobs.<name>.notification_settings
+### jobs.\<name\>.notification_settings
   
 **`Type: Map`**
   
@@ -2791,7 +2795,7 @@ Optional notification settings that are used when sending notifications to each
      - If true, do not send notifications to recipients specified in `on_failure` if the run is skipped.
   
   
-### jobs.<name>.parameters
+### jobs.\<name\>.parameters
   
 **`Type: Sequence`**
   
@@ -2815,7 +2819,7 @@ Job-level parameter definitions
      - The name of the defined parameter. May only contain alphanumeric characters, `_`, `-`, and `.`
   
   
-### jobs.<name>.permissions
+### jobs.\<name\>.permissions
   
 **`Type: Sequence`**
   
@@ -2847,7 +2851,7 @@ Job-level parameter definitions
      - The name of the user that has the permission set in level.
   
   
-### jobs.<name>.queue
+### jobs.\<name\>.queue
   
 **`Type: Map`**
   
@@ -2867,7 +2871,7 @@ The queue settings of the job.
      - If true, enable queueing for the job. This is a required field.
   
   
-### jobs.<name>.run_as
+### jobs.\<name\>.run_as
   
 **`Type: Map`**
   
@@ -2893,7 +2897,7 @@ Either `user_name` or `service_principal_name` should be specified. If not, an e
      - The email of an active workspace user. Non-admin users can only set this field to their own email.
   
   
-### jobs.<name>.schedule
+### jobs.\<name\>.schedule
   
 **`Type: Map`**
   
@@ -2921,7 +2925,7 @@ An optional periodic schedule for this job. The default behavior is that the job
      - A Java timezone ID. The schedule for a job is resolved with respect to this timezone. See [Java TimeZone](https://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html) for details. This field is required.
   
   
-### jobs.<name>.tasks
+### jobs.\<name\>.tasks
   
 **`Type: Sequence`**
   
@@ -2939,19 +2943,19 @@ If more than 100 tasks are available, you can paginate through them using :metho
   
    * - `clean_rooms_notebook_task`
      - Map
-     - The task runs a [clean rooms](https://docs.databricks.com/en/clean-rooms/index.html) notebook when the `clean_rooms_notebook_task` field is present. See [_](#jobs.<name>.tasks.clean_rooms_notebook_task).
+     - The task runs a [clean rooms](https://docs.databricks.com/en/clean-rooms/index.html) notebook when the `clean_rooms_notebook_task` field is present. See [_](#jobsnametasksclean_rooms_notebook_task).
   
    * - `condition_task`
      - Map
-     - The task evaluates a condition that can be used to control the execution of other tasks when the `condition_task` field is present. The condition task does not require a cluster to execute and does not support retries or notifications. See [_](#jobs.<name>.tasks.condition_task).
+     - The task evaluates a condition that can be used to control the execution of other tasks when the `condition_task` field is present. The condition task does not require a cluster to execute and does not support retries or notifications. See [_](#jobsnametaskscondition_task).
   
    * - `dbt_task`
      - Map
-     - The task runs one or more dbt commands when the `dbt_task` field is present. The dbt task requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse. See [_](#jobs.<name>.tasks.dbt_task).
+     - The task runs one or more dbt commands when the `dbt_task` field is present. The dbt task requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse. See [_](#jobsnametasksdbt_task).
   
    * - `depends_on`
      - Sequence
-     - An optional array of objects specifying the dependency graph of the task. All tasks specified in this field must complete before executing this task. The task will run only if the `run_if` condition is true. The key is `task_key`, and the value is the name assigned to the dependent task. See [_](#jobs.<name>.tasks.depends_on).
+     - An optional array of objects specifying the dependency graph of the task. All tasks specified in this field must complete before executing this task. The task will run only if the `run_if` condition is true. The key is `task_key`, and the value is the name assigned to the dependent task. See [_](#jobsnametasksdepends_on).
   
    * - `description`
      - String
@@ -2963,7 +2967,7 @@ If more than 100 tasks are available, you can paginate through them using :metho
   
    * - `email_notifications`
      - Map
-     - An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails. See [_](#jobs.<name>.tasks.email_notifications).
+     - An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails. See [_](#jobsnametasksemail_notifications).
   
    * - `environment_key`
      - String
@@ -2975,11 +2979,11 @@ If more than 100 tasks are available, you can paginate through them using :metho
   
    * - `for_each_task`
      - Map
-     - The task executes a nested task for every input provided when the `for_each_task` field is present. See [_](#jobs.<name>.tasks.for_each_task).
+     - The task executes a nested task for every input provided when the `for_each_task` field is present. See [_](#jobsnametasksfor_each_task).
   
    * - `health`
      - Map
-     - An optional set of health rules that can be defined for this job. See [_](#jobs.<name>.tasks.health).
+     - An optional set of health rules that can be defined for this job. See [_](#jobsnametaskshealth).
   
    * - `job_cluster_key`
      - String
@@ -2987,7 +2991,7 @@ If more than 100 tasks are available, you can paginate through them using :metho
   
    * - `libraries`
      - Sequence
-     - An optional list of libraries to be installed on the cluster. The default value is an empty list. See [_](#jobs.<name>.tasks.libraries).
+     - An optional list of libraries to be installed on the cluster. The default value is an empty list. See [_](#jobsnametaskslibraries).
   
    * - `max_retries`
      - Integer
@@ -2999,23 +3003,23 @@ If more than 100 tasks are available, you can paginate through them using :metho
   
    * - `new_cluster`
      - Map
-     - If new_cluster, a description of a new cluster that is created for each run. See [_](#jobs.<name>.tasks.new_cluster).
+     - If new_cluster, a description of a new cluster that is created for each run. See [_](#jobsnametasksnew_cluster).
   
    * - `notebook_task`
      - Map
-     - The task runs a notebook when the `notebook_task` field is present. See [_](#jobs.<name>.tasks.notebook_task).
+     - The task runs a notebook when the `notebook_task` field is present. See [_](#jobsnametasksnotebook_task).
   
    * - `notification_settings`
      - Map
-     - Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this task. See [_](#jobs.<name>.tasks.notification_settings).
+     - Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this task. See [_](#jobsnametasksnotification_settings).
   
    * - `pipeline_task`
      - Map
-     - The task triggers a pipeline update when the `pipeline_task` field is present. Only pipelines configured to use triggered more are supported. See [_](#jobs.<name>.tasks.pipeline_task).
+     - The task triggers a pipeline update when the `pipeline_task` field is present. Only pipelines configured to use triggered more are supported. See [_](#jobsnametaskspipeline_task).
   
    * - `python_wheel_task`
      - Map
-     - The task runs a Python wheel when the `python_wheel_task` field is present. See [_](#jobs.<name>.tasks.python_wheel_task).
+     - The task runs a Python wheel when the `python_wheel_task` field is present. See [_](#jobsnametaskspython_wheel_task).
   
    * - `retry_on_timeout`
      - Boolean
@@ -3027,23 +3031,23 @@ If more than 100 tasks are available, you can paginate through them using :metho
   
    * - `run_job_task`
      - Map
-     - The task triggers another job when the `run_job_task` field is present. See [_](#jobs.<name>.tasks.run_job_task).
+     - The task triggers another job when the `run_job_task` field is present. See [_](#jobsnametasksrun_job_task).
   
    * - `spark_jar_task`
      - Map
-     - The task runs a JAR when the `spark_jar_task` field is present. See [_](#jobs.<name>.tasks.spark_jar_task).
+     - The task runs a JAR when the `spark_jar_task` field is present. See [_](#jobsnametasksspark_jar_task).
   
    * - `spark_python_task`
      - Map
-     - The task runs a Python file when the `spark_python_task` field is present. See [_](#jobs.<name>.tasks.spark_python_task).
+     - The task runs a Python file when the `spark_python_task` field is present. See [_](#jobsnametasksspark_python_task).
   
    * - `spark_submit_task`
      - Map
-     - (Legacy) The task runs the spark-submit script when the `spark_submit_task` field is present. This task can run only on new clusters and is not compatible with serverless compute.  In the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark configurations.  `master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you _cannot_ specify them in parameters.  By default, the Spark submit job uses all available memory (excluding reserved memory for Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value to leave some room for off-heap usage.  The `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths. See [_](#jobs.<name>.tasks.spark_submit_task).
+     - (Legacy) The task runs the spark-submit script when the `spark_submit_task` field is present. This task can run only on new clusters and is not compatible with serverless compute.  In the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark configurations.  `master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you _cannot_ specify them in parameters.  By default, the Spark submit job uses all available memory (excluding reserved memory for Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value to leave some room for off-heap usage.  The `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths. See [_](#jobsnametasksspark_submit_task).
   
    * - `sql_task`
      - Map
-     - The task runs a SQL query or file, or it refreshes a SQL alert or a legacy SQL dashboard when the `sql_task` field is present. See [_](#jobs.<name>.tasks.sql_task).
+     - The task runs a SQL query or file, or it refreshes a SQL alert or a legacy SQL dashboard when the `sql_task` field is present. See [_](#jobsnametaskssql_task).
   
    * - `task_key`
      - String
@@ -3055,10 +3059,10 @@ If more than 100 tasks are available, you can paginate through them using :metho
   
    * - `webhook_notifications`
      - Map
-     - A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications. See [_](#jobs.<name>.tasks.webhook_notifications).
+     - A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications. See [_](#jobsnametaskswebhook_notifications).
   
   
-### jobs.<name>.tasks.clean_rooms_notebook_task
+### jobs.\<name\>.tasks.clean_rooms_notebook_task
   
 **`Type: Map`**
   
@@ -3091,7 +3095,7 @@ when the `clean_rooms_notebook_task` field is present.
      - Name of the notebook being run.
   
   
-### jobs.<name>.tasks.condition_task
+### jobs.\<name\>.tasks.condition_task
   
 **`Type: Map`**
   
@@ -3120,7 +3124,7 @@ The condition task does not require a cluster to execute and does not support re
      - The right operand of the condition task. Can be either a string value or a job state or parameter reference.
   
   
-### jobs.<name>.tasks.dbt_task
+### jobs.\<name\>.tasks.dbt_task
   
 **`Type: Map`**
   
@@ -3164,7 +3168,7 @@ The task runs one or more dbt commands when the `dbt_task` field is present. The
      - ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the profile and connection details to dbt. It can be overridden on a per-command basis by using the `--profiles-dir` command line argument.
   
   
-### jobs.<name>.tasks.depends_on
+### jobs.\<name\>.tasks.depends_on
   
 **`Type: Sequence`**
   
@@ -3189,7 +3193,7 @@ The key is `task_key`, and the value is the name assigned to the dependent task.
      - The name of the task this task depends on.
   
   
-### jobs.<name>.tasks.email_notifications
+### jobs.\<name\>.tasks.email_notifications
   
 **`Type: Map`**
   
@@ -3229,7 +3233,7 @@ An optional set of email addresses that is notified when runs of this task begin
      - A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.
   
   
-### jobs.<name>.tasks.for_each_task
+### jobs.\<name\>.tasks.for_each_task
   
 **`Type: Map`**
   
@@ -3257,7 +3261,7 @@ The task executes a nested task for every input provided when the `for_each_task
      - Configuration for the task that will be run for each element in the array
   
   
-### jobs.<name>.tasks.health
+### jobs.\<name\>.tasks.health
   
 **`Type: Map`**
   
@@ -3274,10 +3278,10 @@ An optional set of health rules that can be defined for this job.
   
    * - `rules`
      - Sequence
-     - See [_](#jobs.<name>.tasks.health.rules).
+     - See [_](#jobsnametaskshealthrules).
   
   
-### jobs.<name>.tasks.health.rules
+### jobs.\<name\>.tasks.health.rules
   
 **`Type: Sequence`**
   
@@ -3305,7 +3309,7 @@ An optional set of health rules that can be defined for this job.
      - Specifies the threshold value that the health metric should obey to satisfy the health rule.
   
   
-### jobs.<name>.tasks.libraries
+### jobs.\<name\>.tasks.libraries
   
 **`Type: Sequence`**
   
@@ -3323,7 +3327,7 @@ The default value is an empty list.
   
    * - `cran`
      - Map
-     - Specification of a CRAN library to be installed as part of the library. See [_](#jobs.<name>.tasks.libraries.cran).
+     - Specification of a CRAN library to be installed as part of the library. See [_](#jobsnametaskslibrariescran).
   
    * - `egg`
      - String
@@ -3335,11 +3339,11 @@ The default value is an empty list.
   
    * - `maven`
      - Map
-     - Specification of a maven library to be installed. For example: `{ "coordinates": "org.jsoup:jsoup:1.7.2" }`. See [_](#jobs.<name>.tasks.libraries.maven).
+     - Specification of a maven library to be installed. For example: `{ "coordinates": "org.jsoup:jsoup:1.7.2" }`. See [_](#jobsnametaskslibrariesmaven).
   
    * - `pypi`
      - Map
-     - Specification of a PyPi library to be installed. For example: `{ "package": "simplejson" }`. See [_](#jobs.<name>.tasks.libraries.pypi).
+     - Specification of a PyPi library to be installed. For example: `{ "package": "simplejson" }`. See [_](#jobsnametaskslibrariespypi).
   
    * - `requirements`
      - String
@@ -3350,7 +3354,7 @@ The default value is an empty list.
      - URI of the wheel library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs. For example: `{ "whl": "/Workspace/path/to/library.whl" }`, `{ "whl" : "/Volumes/path/to/library.whl" }` or `{ "whl": "s3://my-bucket/library.whl" }`. If S3 is used, please make sure the cluster has read access on the library. You may need to launch the cluster with an IAM role to access the S3 URI.
   
   
-### jobs.<name>.tasks.libraries.cran
+### jobs.\<name\>.tasks.libraries.cran
   
 **`Type: Map`**
   
@@ -3374,7 +3378,7 @@ Specification of a CRAN library to be installed as part of the library
      - The repository where the package can be found. If not specified, the default CRAN repo is used.
   
   
-### jobs.<name>.tasks.libraries.maven
+### jobs.\<name\>.tasks.libraries.maven
   
 **`Type: Map`**
   
@@ -3403,7 +3407,7 @@ Specification of a maven library to be installed. For example:
      - Maven repo to install the Maven package from. If omitted, both Maven Central Repository and Spark Packages are searched.
   
   
-### jobs.<name>.tasks.libraries.pypi
+### jobs.\<name\>.tasks.libraries.pypi
   
 **`Type: Map`**
   
@@ -3428,7 +3432,7 @@ Specification of a PyPi library to be installed. For example:
      - The repository where the package can be found. If not specified, the default pip index is used.
   
   
-### jobs.<name>.tasks.new_cluster
+### jobs.\<name\>.tasks.new_cluster
   
 **`Type: Map`**
   
@@ -3449,7 +3453,7 @@ If new_cluster, a description of a new cluster that is created for each run.
   
    * - `autoscale`
      - Map
-     - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#jobs.<name>.tasks.new_cluster.autoscale).
+     - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#jobsnametasksnew_clusterautoscale).
   
    * - `autotermination_minutes`
      - Integer
@@ -3457,15 +3461,15 @@ If new_cluster, a description of a new cluster that is created for each run.
   
    * - `aws_attributes`
      - Map
-     - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#jobs.<name>.tasks.new_cluster.aws_attributes).
+     - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#jobsnametasksnew_clusteraws_attributes).
   
    * - `azure_attributes`
      - Map
-     - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#jobs.<name>.tasks.new_cluster.azure_attributes).
+     - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#jobsnametasksnew_clusterazure_attributes).
   
    * - `cluster_log_conf`
      - Map
-     - The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. See [_](#jobs.<name>.tasks.new_cluster.cluster_log_conf).
+     - The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. See [_](#jobsnametasksnew_clustercluster_log_conf).
   
    * - `cluster_name`
      - String
@@ -3481,7 +3485,7 @@ If new_cluster, a description of a new cluster that is created for each run.
   
    * - `docker_image`
      - Map
-     - See [_](#jobs.<name>.tasks.new_cluster.docker_image).
+     - See [_](#jobsnametasksnew_clusterdocker_image).
   
    * - `driver_instance_pool_id`
      - String
@@ -3501,11 +3505,11 @@ If new_cluster, a description of a new cluster that is created for each run.
   
    * - `gcp_attributes`
      - Map
-     - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#jobs.<name>.tasks.new_cluster.gcp_attributes).
+     - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#jobsnametasksnew_clustergcp_attributes).
   
    * - `init_scripts`
      - Sequence
-     - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `<destination>/<cluster-ID>/init_scripts`. See [_](#jobs.<name>.tasks.new_cluster.init_scripts).
+     - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `<destination>/<cluster-ID>/init_scripts`. See [_](#jobsnametasksnew_clusterinit_scripts).
   
    * - `instance_pool_id`
      - String
@@ -3561,10 +3565,10 @@ If new_cluster, a description of a new cluster that is created for each run.
   
    * - `workload_type`
      - Map
-     - See [_](#jobs.<name>.tasks.new_cluster.workload_type).
+     - See [_](#jobsnametasksnew_clusterworkload_type).
   
   
-### jobs.<name>.tasks.new_cluster.autoscale
+### jobs.\<name\>.tasks.new_cluster.autoscale
   
 **`Type: Map`**
   
@@ -3589,7 +3593,7 @@ Note: autoscaling works best with DB runtime versions 3.0 or later.
      - The minimum number of workers to which the cluster can scale down when underutilized. It is also the initial number of workers the cluster will have after creation.
   
   
-### jobs.<name>.tasks.new_cluster.aws_attributes
+### jobs.\<name\>.tasks.new_cluster.aws_attributes
   
 **`Type: Map`**
   
@@ -3646,7 +3650,7 @@ If not specified at cluster creation, a set of default values will be used.
      - Identifier for the availability zone/datacenter in which the cluster resides. This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. This is an optional field at cluster creation, and if not specified, a default zone will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. The list of available zones as well as the default value can be found by using the `List Zones` method.
   
   
-### jobs.<name>.tasks.new_cluster.azure_attributes
+### jobs.\<name\>.tasks.new_cluster.azure_attributes
   
 **`Type: Map`**
   
@@ -3672,14 +3676,14 @@ If not specified at cluster creation, a set of default values will be used.
   
    * - `log_analytics_info`
      - Map
-     - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#jobs.<name>.tasks.new_cluster.azure_attributes.log_analytics_info).
+     - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#jobsnametasksnew_clusterazure_attributeslog_analytics_info).
   
    * - `spot_bid_max_price`
      - Any
      - The max bid price to be used for Azure spot instances. The Max price for the bid cannot be higher than the on-demand price of the instance. If not specified, the default value is -1, which specifies that the instance cannot be evicted on the basis of price, and only on the basis of availability. Further, the value should > 0 or -1.
   
   
-### jobs.<name>.tasks.new_cluster.azure_attributes.log_analytics_info
+### jobs.\<name\>.tasks.new_cluster.azure_attributes.log_analytics_info
   
 **`Type: Map`**
   
@@ -3703,7 +3707,7 @@ Defines values necessary to configure and run Azure Log Analytics agent
      - <needs content added>
   
   
-### jobs.<name>.tasks.new_cluster.cluster_log_conf
+### jobs.\<name\>.tasks.new_cluster.cluster_log_conf
   
 **`Type: Map`**
   
@@ -3724,14 +3728,14 @@ the destination of executor logs is `$destination/$clusterId/executor`.
   
    * - `dbfs`
      - Map
-     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobs.<name>.tasks.new_cluster.cluster_log_conf.dbfs).
+     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobsnametasksnew_clustercluster_log_confdbfs).
   
    * - `s3`
      - Map
-     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobs.<name>.tasks.new_cluster.cluster_log_conf.s3).
+     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobsnametasksnew_clustercluster_log_confs3).
   
   
-### jobs.<name>.tasks.new_cluster.cluster_log_conf.dbfs
+### jobs.\<name\>.tasks.new_cluster.cluster_log_conf.dbfs
   
 **`Type: Map`**
   
@@ -3752,7 +3756,7 @@ destination needs to be provided. e.g.
      - dbfs destination, e.g. `dbfs:/my/path`
   
   
-### jobs.<name>.tasks.new_cluster.cluster_log_conf.s3
+### jobs.\<name\>.tasks.new_cluster.cluster_log_conf.s3
   
 **`Type: Map`**
   
@@ -3799,7 +3803,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in
      - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
   
   
-### jobs.<name>.tasks.new_cluster.docker_image
+### jobs.\<name\>.tasks.new_cluster.docker_image
   
 **`Type: Map`**
   
@@ -3816,14 +3820,14 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in
   
    * - `basic_auth`
      - Map
-     - See [_](#jobs.<name>.tasks.new_cluster.docker_image.basic_auth).
+     - See [_](#jobsnametasksnew_clusterdocker_imagebasic_auth).
   
    * - `url`
      - String
      - URL of the docker image.
   
   
-### jobs.<name>.tasks.new_cluster.docker_image.basic_auth
+### jobs.\<name\>.tasks.new_cluster.docker_image.basic_auth
   
 **`Type: Map`**
   
@@ -3847,7 +3851,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in
      - Name of the user
   
   
-### jobs.<name>.tasks.new_cluster.gcp_attributes
+### jobs.\<name\>.tasks.new_cluster.gcp_attributes
   
 **`Type: Map`**
   
@@ -3888,7 +3892,7 @@ If not specified at cluster creation, a set of default values will be used.
      - Identifier for the availability zone in which the cluster resides. This can be one of the following: - "HA" => High availability, spread nodes across availability zones for a Databricks deployment region [default] - "AUTO" => Databricks picks an availability zone to schedule the cluster on. - A GCP availability zone => Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones.
   
   
-### jobs.<name>.tasks.new_cluster.init_scripts
+### jobs.\<name\>.tasks.new_cluster.init_scripts
   
 **`Type: Sequence`**
   
@@ -3905,34 +3909,34 @@ The configuration for storing init scripts. Any number of destinations can be sp
   
    * - `abfss`
      - Map
-     - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>" } }. See [_](#jobs.<name>.tasks.new_cluster.init_scripts.abfss).
+     - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>" } }. See [_](#jobsnametasksnew_clusterinit_scriptsabfss).
   
    * - `dbfs`
      - Map
-     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobs.<name>.tasks.new_cluster.init_scripts.dbfs).
+     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobsnametasksnew_clusterinit_scriptsdbfs).
   
    * - `file`
      - Map
-     - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#jobs.<name>.tasks.new_cluster.init_scripts.file).
+     - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#jobsnametasksnew_clusterinit_scriptsfile).
   
    * - `gcs`
      - Map
-     - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#jobs.<name>.tasks.new_cluster.init_scripts.gcs).
+     - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#jobsnametasksnew_clusterinit_scriptsgcs).
   
    * - `s3`
      - Map
-     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobs.<name>.tasks.new_cluster.init_scripts.s3).
+     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobsnametasksnew_clusterinit_scriptss3).
   
    * - `volumes`
      - Map
-     - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#jobs.<name>.tasks.new_cluster.init_scripts.volumes).
+     - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#jobsnametasksnew_clusterinit_scriptsvolumes).
   
    * - `workspace`
      - Map
-     - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#jobs.<name>.tasks.new_cluster.init_scripts.workspace).
+     - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#jobsnametasksnew_clusterinit_scriptsworkspace).
   
   
-### jobs.<name>.tasks.new_cluster.init_scripts.abfss
+### jobs.\<name\>.tasks.new_cluster.init_scripts.abfss
   
 **`Type: Map`**
   
@@ -3953,7 +3957,7 @@ destination needs to be provided. e.g.
      - abfss destination, e.g. `abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>`.
   
   
-### jobs.<name>.tasks.new_cluster.init_scripts.dbfs
+### jobs.\<name\>.tasks.new_cluster.init_scripts.dbfs
   
 **`Type: Map`**
   
@@ -3974,7 +3978,7 @@ destination needs to be provided. e.g.
      - dbfs destination, e.g. `dbfs:/my/path`
   
   
-### jobs.<name>.tasks.new_cluster.init_scripts.file
+### jobs.\<name\>.tasks.new_cluster.init_scripts.file
   
 **`Type: Map`**
   
@@ -3995,7 +3999,7 @@ destination needs to be provided. e.g.
      - local file destination, e.g. `file:/my/local/file.sh`
   
   
-### jobs.<name>.tasks.new_cluster.init_scripts.gcs
+### jobs.\<name\>.tasks.new_cluster.init_scripts.gcs
   
 **`Type: Map`**
   
@@ -4016,7 +4020,7 @@ destination needs to be provided. e.g.
      - GCS destination/URI, e.g. `gs://my-bucket/some-prefix`
   
   
-### jobs.<name>.tasks.new_cluster.init_scripts.s3
+### jobs.\<name\>.tasks.new_cluster.init_scripts.s3
   
 **`Type: Map`**
   
@@ -4063,7 +4067,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in
      - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
   
   
-### jobs.<name>.tasks.new_cluster.init_scripts.volumes
+### jobs.\<name\>.tasks.new_cluster.init_scripts.volumes
   
 **`Type: Map`**
   
@@ -4084,7 +4088,7 @@ destination needs to be provided. e.g.
      - Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`
   
   
-### jobs.<name>.tasks.new_cluster.init_scripts.workspace
+### jobs.\<name\>.tasks.new_cluster.init_scripts.workspace
   
 **`Type: Map`**
   
@@ -4105,7 +4109,7 @@ destination needs to be provided. e.g.
      - workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`
   
   
-### jobs.<name>.tasks.new_cluster.workload_type
+### jobs.\<name\>.tasks.new_cluster.workload_type
   
 **`Type: Map`**
   
@@ -4122,10 +4126,10 @@ destination needs to be provided. e.g.
   
    * - `clients`
      - Map
-     -  defined what type of clients can use the cluster. E.g. Notebooks, Jobs. See [_](#jobs.<name>.tasks.new_cluster.workload_type.clients).
+     -  defined what type of clients can use the cluster. E.g. Notebooks, Jobs. See [_](#jobsnametasksnew_clusterworkload_typeclients).
   
   
-### jobs.<name>.tasks.new_cluster.workload_type.clients
+### jobs.\<name\>.tasks.new_cluster.workload_type.clients
   
 **`Type: Map`**
   
@@ -4149,7 +4153,7 @@ destination needs to be provided. e.g.
      - With notebooks set, this cluster can be used for notebooks
   
   
-### jobs.<name>.tasks.notebook_task
+### jobs.\<name\>.tasks.notebook_task
   
 **`Type: Map`**
   
@@ -4181,7 +4185,7 @@ The task runs a notebook when the `notebook_task` field is present.
      - Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses.  Note that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail.
   
   
-### jobs.<name>.tasks.notification_settings
+### jobs.\<name\>.tasks.notification_settings
   
 **`Type: Map`**
   
@@ -4209,7 +4213,7 @@ Optional notification settings that are used when sending notifications to each
      - If true, do not send notifications to recipients specified in `on_failure` if the run is skipped.
   
   
-### jobs.<name>.tasks.pipeline_task
+### jobs.\<name\>.tasks.pipeline_task
   
 **`Type: Map`**
   
@@ -4233,7 +4237,7 @@ The task triggers a pipeline update when the `pipeline_task` field is present. O
      - The full name of the pipeline task to execute.
   
   
-### jobs.<name>.tasks.python_wheel_task
+### jobs.\<name\>.tasks.python_wheel_task
   
 **`Type: Map`**
   
@@ -4265,7 +4269,7 @@ The task runs a Python wheel when the `python_wheel_task` field is present.
      - Command-line parameters passed to Python wheel task. Leave it empty if `named_parameters` is not null.
   
   
-### jobs.<name>.tasks.run_job_task
+### jobs.\<name\>.tasks.run_job_task
   
 **`Type: Map`**
   
@@ -4302,7 +4306,7 @@ The task triggers another job when the `run_job_task` field is present.
   
    * - `pipeline_params`
      - Map
-     - Controls whether the pipeline should perform a full refresh. See [_](#jobs.<name>.tasks.run_job_task.pipeline_params).
+     - Controls whether the pipeline should perform a full refresh. See [_](#jobsnametasksrun_job_taskpipeline_params).
   
    * - `python_named_params`
      - Map
@@ -4321,7 +4325,7 @@ The task triggers another job when the `run_job_task` field is present.
      - A map from keys to values for jobs with SQL task, for example `"sql_params": {"name": "john doe", "age": "35"}`. The SQL alert task does not support custom parameters.
   
   
-### jobs.<name>.tasks.run_job_task.pipeline_params
+### jobs.\<name\>.tasks.run_job_task.pipeline_params
   
 **`Type: Map`**
   
@@ -4341,7 +4345,7 @@ Controls whether the pipeline should perform a full refresh
      - If true, triggers a full refresh on the delta live table.
   
   
-### jobs.<name>.tasks.spark_jar_task
+### jobs.\<name\>.tasks.spark_jar_task
   
 **`Type: Map`**
   
@@ -4373,7 +4377,7 @@ The task runs a JAR when the `spark_jar_task` field is present.
      - Deprecated. A value of `false` is no longer supported.
   
   
-### jobs.<name>.tasks.spark_python_task
+### jobs.\<name\>.tasks.spark_python_task
   
 **`Type: Map`**
   
@@ -4401,7 +4405,7 @@ The task runs a Python file when the `spark_python_task` field is present.
      - Optional location type of the Python file. When set to `WORKSPACE` or not specified, the file will be retrieved from the local Databricks workspace or cloud location (if the `python_file` has a URI format). When set to `GIT`, the Python file will be retrieved from a Git repository defined in `git_source`.  * `WORKSPACE`: The Python file is located in a Databricks workspace or at a cloud filesystem URI. * `GIT`: The Python file is located in a remote Git repository.
   
   
-### jobs.<name>.tasks.spark_submit_task
+### jobs.\<name\>.tasks.spark_submit_task
   
 **`Type: Map`**
   
@@ -4429,7 +4433,7 @@ The `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths.
      - Command-line parameters passed to spark submit.  Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.
   
   
-### jobs.<name>.tasks.sql_task
+### jobs.\<name\>.tasks.sql_task
   
 **`Type: Map`**
   
@@ -4446,15 +4450,15 @@ The task runs a SQL query or file, or it refreshes a SQL alert or a legacy SQL d
   
    * - `alert`
      - Map
-     - If alert, indicates that this job must refresh a SQL alert. See [_](#jobs.<name>.tasks.sql_task.alert).
+     - If alert, indicates that this job must refresh a SQL alert. See [_](#jobsnametaskssql_taskalert).
   
    * - `dashboard`
      - Map
-     - If dashboard, indicates that this job must refresh a SQL dashboard. See [_](#jobs.<name>.tasks.sql_task.dashboard).
+     - If dashboard, indicates that this job must refresh a SQL dashboard. See [_](#jobsnametaskssql_taskdashboard).
   
    * - `file`
      - Map
-     - If file, indicates that this job runs a SQL file in a remote Git repository. See [_](#jobs.<name>.tasks.sql_task.file).
+     - If file, indicates that this job runs a SQL file in a remote Git repository. See [_](#jobsnametaskssql_taskfile).
   
    * - `parameters`
      - Map
@@ -4462,14 +4466,14 @@ The task runs a SQL query or file, or it refreshes a SQL alert or a legacy SQL d
   
    * - `query`
      - Map
-     - If query, indicates that this job must execute a SQL query. See [_](#jobs.<name>.tasks.sql_task.query).
+     - If query, indicates that this job must execute a SQL query. See [_](#jobsnametaskssql_taskquery).
   
    * - `warehouse_id`
      - String
      - The canonical identifier of the SQL warehouse. Recommended to use with serverless or pro SQL warehouses. Classic SQL warehouses are only supported for SQL alert, dashboard and query tasks and are limited to scheduled single-task jobs.
   
   
-### jobs.<name>.tasks.sql_task.alert
+### jobs.\<name\>.tasks.sql_task.alert
   
 **`Type: Map`**
   
@@ -4494,10 +4498,10 @@ If alert, indicates that this job must refresh a SQL alert.
   
    * - `subscriptions`
      - Sequence
-     - If specified, alert notifications are sent to subscribers. See [_](#jobs.<name>.tasks.sql_task.alert.subscriptions).
+     - If specified, alert notifications are sent to subscribers. See [_](#jobsnametaskssql_taskalertsubscriptions).
   
   
-### jobs.<name>.tasks.sql_task.alert.subscriptions
+### jobs.\<name\>.tasks.sql_task.alert.subscriptions
   
 **`Type: Sequence`**
   
@@ -4521,7 +4525,7 @@ If specified, alert notifications are sent to subscribers.
      - The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications.
   
   
-### jobs.<name>.tasks.sql_task.dashboard
+### jobs.\<name\>.tasks.sql_task.dashboard
   
 **`Type: Map`**
   
@@ -4550,10 +4554,10 @@ If dashboard, indicates that this job must refresh a SQL dashboard.
   
    * - `subscriptions`
      - Sequence
-     - If specified, dashboard snapshots are sent to subscriptions. See [_](#jobs.<name>.tasks.sql_task.dashboard.subscriptions).
+     - If specified, dashboard snapshots are sent to subscriptions. See [_](#jobsnametaskssql_taskdashboardsubscriptions).
   
   
-### jobs.<name>.tasks.sql_task.dashboard.subscriptions
+### jobs.\<name\>.tasks.sql_task.dashboard.subscriptions
   
 **`Type: Sequence`**
   
@@ -4577,7 +4581,7 @@ If specified, dashboard snapshots are sent to subscriptions.
      - The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications.
   
   
-### jobs.<name>.tasks.sql_task.file
+### jobs.\<name\>.tasks.sql_task.file
   
 **`Type: Map`**
   
@@ -4601,7 +4605,7 @@ If file, indicates that this job runs a SQL file in a remote Git repository.
      - Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved from the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository defined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.  * `WORKSPACE`: SQL file is located in Databricks workspace. * `GIT`: SQL file is located in cloud Git provider.
   
   
-### jobs.<name>.tasks.sql_task.query
+### jobs.\<name\>.tasks.sql_task.query
   
 **`Type: Map`**
   
@@ -4621,7 +4625,7 @@ If query, indicates that this job must execute a SQL query.
      - The canonical identifier of the SQL query.
   
   
-### jobs.<name>.tasks.webhook_notifications
+### jobs.\<name\>.tasks.webhook_notifications
   
 **`Type: Map`**
   
@@ -4638,26 +4642,26 @@ A collection of system notification IDs to notify when runs of this task begin o
   
    * - `on_duration_warning_threshold_exceeded`
      - Sequence
-     - An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property. See [_](#jobs.<name>.tasks.webhook_notifications.on_duration_warning_threshold_exceeded).
+     - An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property. See [_](#jobsnametaskswebhook_notificationson_duration_warning_threshold_exceeded).
   
    * - `on_failure`
      - Sequence
-     - An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property. See [_](#jobs.<name>.tasks.webhook_notifications.on_failure).
+     - An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property. See [_](#jobsnametaskswebhook_notificationson_failure).
   
    * - `on_start`
      - Sequence
-     - An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property. See [_](#jobs.<name>.tasks.webhook_notifications.on_start).
+     - An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property. See [_](#jobsnametaskswebhook_notificationson_start).
   
    * - `on_streaming_backlog_exceeded`
      - Sequence
-     - An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes. A maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property. See [_](#jobs.<name>.tasks.webhook_notifications.on_streaming_backlog_exceeded).
+     - An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes. A maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property. See [_](#jobsnametaskswebhook_notificationson_streaming_backlog_exceeded).
   
    * - `on_success`
      - Sequence
-     - An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property. See [_](#jobs.<name>.tasks.webhook_notifications.on_success).
+     - An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property. See [_](#jobsnametaskswebhook_notificationson_success).
   
   
-### jobs.<name>.tasks.webhook_notifications.on_duration_warning_threshold_exceeded
+### jobs.\<name\>.tasks.webhook_notifications.on_duration_warning_threshold_exceeded
   
 **`Type: Sequence`**
   
@@ -4677,7 +4681,7 @@ An optional list of system notification IDs to call when the duration of a run e
      - 
   
   
-### jobs.<name>.tasks.webhook_notifications.on_failure
+### jobs.\<name\>.tasks.webhook_notifications.on_failure
   
 **`Type: Sequence`**
   
@@ -4697,7 +4701,7 @@ An optional list of system notification IDs to call when the run fails. A maximu
      - 
   
   
-### jobs.<name>.tasks.webhook_notifications.on_start
+### jobs.\<name\>.tasks.webhook_notifications.on_start
   
 **`Type: Sequence`**
   
@@ -4717,7 +4721,7 @@ An optional list of system notification IDs to call when the run starts. A maxim
      - 
   
   
-### jobs.<name>.tasks.webhook_notifications.on_streaming_backlog_exceeded
+### jobs.\<name\>.tasks.webhook_notifications.on_streaming_backlog_exceeded
   
 **`Type: Sequence`**
   
@@ -4740,7 +4744,7 @@ A maximum of 3 destinations can be specified for the `on_streaming_backlog_excee
      - 
   
   
-### jobs.<name>.tasks.webhook_notifications.on_success
+### jobs.\<name\>.tasks.webhook_notifications.on_success
   
 **`Type: Sequence`**
   
@@ -4760,7 +4764,7 @@ An optional list of system notification IDs to call when the run completes succe
      - 
   
   
-### jobs.<name>.trigger
+### jobs.\<name\>.trigger
   
 **`Type: Map`**
   
@@ -4777,7 +4781,7 @@ A configuration to trigger a run when certain conditions are met. The default be
   
    * - `file_arrival`
      - Map
-     - File arrival trigger settings. See [_](#jobs.<name>.trigger.file_arrival).
+     - File arrival trigger settings. See [_](#jobsnametriggerfile_arrival).
   
    * - `pause_status`
      - String
@@ -4785,18 +4789,18 @@ A configuration to trigger a run when certain conditions are met. The default be
   
    * - `periodic`
      - Map
-     - Periodic trigger settings. See [_](#jobs.<name>.trigger.periodic).
+     - Periodic trigger settings. See [_](#jobsnametriggerperiodic).
   
    * - `table`
      - Map
-     - Old table trigger settings name. Deprecated in favor of `table_update`. See [_](#jobs.<name>.trigger.table).
+     - Old table trigger settings name. Deprecated in favor of `table_update`. See [_](#jobsnametriggertable).
   
    * - `table_update`
      - Map
-     - See [_](#jobs.<name>.trigger.table_update).
+     - See [_](#jobsnametriggertable_update).
   
   
-### jobs.<name>.trigger.file_arrival
+### jobs.\<name\>.trigger.file_arrival
   
 **`Type: Map`**
   
@@ -4824,7 +4828,7 @@ File arrival trigger settings.
      - If set, the trigger starts a run only after no file activity has occurred for the specified amount of time. This makes it possible to wait for a batch of incoming files to arrive before triggering a run. The minimum allowed value is 60 seconds.
   
   
-### jobs.<name>.trigger.periodic
+### jobs.\<name\>.trigger.periodic
   
 **`Type: Map`**
   
@@ -4848,7 +4852,7 @@ Periodic trigger settings.
      - The unit of time for the interval.
   
   
-### jobs.<name>.trigger.table
+### jobs.\<name\>.trigger.table
   
 **`Type: Map`**
   
@@ -4880,7 +4884,7 @@ Old table trigger settings name. Deprecated in favor of `table_update`.
      - If set, the trigger starts a run only after no table updates have occurred for the specified time and can be used to wait for a series of table updates before triggering a run. The minimum allowed value is 60 seconds.
   
   
-### jobs.<name>.trigger.table_update
+### jobs.\<name\>.trigger.table_update
   
 **`Type: Map`**
   
@@ -4912,7 +4916,7 @@ Old table trigger settings name. Deprecated in favor of `table_update`.
      - If set, the trigger starts a run only after no table updates have occurred for the specified time and can be used to wait for a series of table updates before triggering a run. The minimum allowed value is 60 seconds.
   
   
-### jobs.<name>.webhook_notifications
+### jobs.\<name\>.webhook_notifications
   
 **`Type: Map`**
   
@@ -4929,26 +4933,26 @@ A collection of system notification IDs to notify when runs of this job begin or
   
    * - `on_duration_warning_threshold_exceeded`
      - Sequence
-     - An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property. See [_](#jobs.<name>.webhook_notifications.on_duration_warning_threshold_exceeded).
+     - An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property. See [_](#jobsnamewebhook_notificationson_duration_warning_threshold_exceeded).
   
    * - `on_failure`
      - Sequence
-     - An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property. See [_](#jobs.<name>.webhook_notifications.on_failure).
+     - An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property. See [_](#jobsnamewebhook_notificationson_failure).
   
    * - `on_start`
      - Sequence
-     - An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property. See [_](#jobs.<name>.webhook_notifications.on_start).
+     - An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property. See [_](#jobsnamewebhook_notificationson_start).
   
    * - `on_streaming_backlog_exceeded`
      - Sequence
-     - An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes. A maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property. See [_](#jobs.<name>.webhook_notifications.on_streaming_backlog_exceeded).
+     - An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes. A maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property. See [_](#jobsnamewebhook_notificationson_streaming_backlog_exceeded).
   
    * - `on_success`
      - Sequence
-     - An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property. See [_](#jobs.<name>.webhook_notifications.on_success).
+     - An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property. See [_](#jobsnamewebhook_notificationson_success).
   
   
-### jobs.<name>.webhook_notifications.on_duration_warning_threshold_exceeded
+### jobs.\<name\>.webhook_notifications.on_duration_warning_threshold_exceeded
   
 **`Type: Sequence`**
   
@@ -4968,7 +4972,7 @@ An optional list of system notification IDs to call when the duration of a run e
      - 
   
   
-### jobs.<name>.webhook_notifications.on_failure
+### jobs.\<name\>.webhook_notifications.on_failure
   
 **`Type: Sequence`**
   
@@ -4988,7 +4992,7 @@ An optional list of system notification IDs to call when the run fails. A maximu
      - 
   
   
-### jobs.<name>.webhook_notifications.on_start
+### jobs.\<name\>.webhook_notifications.on_start
   
 **`Type: Sequence`**
   
@@ -5008,7 +5012,7 @@ An optional list of system notification IDs to call when the run starts. A maxim
      - 
   
   
-### jobs.<name>.webhook_notifications.on_streaming_backlog_exceeded
+### jobs.\<name\>.webhook_notifications.on_streaming_backlog_exceeded
   
 **`Type: Sequence`**
   
@@ -5031,7 +5035,7 @@ A maximum of 3 destinations can be specified for the `on_streaming_backlog_excee
      - 
   
   
-### jobs.<name>.webhook_notifications.on_success
+### jobs.\<name\>.webhook_notifications.on_success
   
 **`Type: Sequence`**
   
@@ -5073,11 +5077,11 @@ model_serving_endpoints:
   
    * - `ai_gateway`
      - Map
-     - The AI Gateway configuration for the serving endpoint. NOTE: Only external model and provisioned throughput endpoints are currently supported. See [_](#model_serving_endpoints.<name>.ai_gateway).
+     - The AI Gateway configuration for the serving endpoint. NOTE: Only external model and provisioned throughput endpoints are currently supported. See [_](#model_serving_endpointsnameai_gateway).
   
    * - `config`
      - Map
-     - The core config of the serving endpoint. See [_](#model_serving_endpoints.<name>.config).
+     - The core config of the serving endpoint. See [_](#model_serving_endpointsnameconfig).
   
    * - `name`
      - String
@@ -5085,11 +5089,11 @@ model_serving_endpoints:
   
    * - `permissions`
      - Sequence
-     - See [_](#model_serving_endpoints.<name>.permissions).
+     - See [_](#model_serving_endpointsnamepermissions).
   
    * - `rate_limits`
      - Sequence
-     - Rate limits to be applied to the serving endpoint. NOTE: this field is deprecated, please use AI Gateway to manage rate limits. See [_](#model_serving_endpoints.<name>.rate_limits).
+     - Rate limits to be applied to the serving endpoint. NOTE: this field is deprecated, please use AI Gateway to manage rate limits. See [_](#model_serving_endpointsnamerate_limits).
   
    * - `route_optimized`
      - Boolean
@@ -5097,7 +5101,7 @@ model_serving_endpoints:
   
    * - `tags`
      - Sequence
-     - Tags to be attached to the serving endpoint and automatically propagated to billing logs. See [_](#model_serving_endpoints.<name>.tags).
+     - Tags to be attached to the serving endpoint and automatically propagated to billing logs. See [_](#model_serving_endpointsnametags).
   
   
 **Example**
@@ -5124,7 +5128,7 @@ resources:
         value: "data science"
 ```
   
-### model_serving_endpoints.<name>.ai_gateway
+### model_serving_endpoints.\<name\>.ai_gateway
   
 **`Type: Map`**
   
@@ -5141,22 +5145,22 @@ The AI Gateway configuration for the serving endpoint. NOTE: Only external model
   
    * - `guardrails`
      - Map
-     - Configuration for AI Guardrails to prevent unwanted data and unsafe data in requests and responses. See [_](#model_serving_endpoints.<name>.ai_gateway.guardrails).
+     - Configuration for AI Guardrails to prevent unwanted data and unsafe data in requests and responses. See [_](#model_serving_endpointsnameai_gatewayguardrails).
   
    * - `inference_table_config`
      - Map
-     - Configuration for payload logging using inference tables. Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality. See [_](#model_serving_endpoints.<name>.ai_gateway.inference_table_config).
+     - Configuration for payload logging using inference tables. Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality. See [_](#model_serving_endpointsnameai_gatewayinference_table_config).
   
    * - `rate_limits`
      - Sequence
-     - Configuration for rate limits which can be set to limit endpoint traffic. See [_](#model_serving_endpoints.<name>.ai_gateway.rate_limits).
+     - Configuration for rate limits which can be set to limit endpoint traffic. See [_](#model_serving_endpointsnameai_gatewayrate_limits).
   
    * - `usage_tracking_config`
      - Map
-     - Configuration to enable usage tracking using system tables. These tables allow you to monitor operational usage on endpoints and their associated costs. See [_](#model_serving_endpoints.<name>.ai_gateway.usage_tracking_config).
+     - Configuration to enable usage tracking using system tables. These tables allow you to monitor operational usage on endpoints and their associated costs. See [_](#model_serving_endpointsnameai_gatewayusage_tracking_config).
   
   
-### model_serving_endpoints.<name>.ai_gateway.guardrails
+### model_serving_endpoints.\<name\>.ai_gateway.guardrails
   
 **`Type: Map`**
   
@@ -5173,14 +5177,14 @@ Configuration for AI Guardrails to prevent unwanted data and unsafe data in requ
   
    * - `input`
      - Map
-     - Configuration for input guardrail filters. See [_](#model_serving_endpoints.<name>.ai_gateway.guardrails.input).
+     - Configuration for input guardrail filters. See [_](#model_serving_endpointsnameai_gatewayguardrailsinput).
   
    * - `output`
      - Map
-     - Configuration for output guardrail filters. See [_](#model_serving_endpoints.<name>.ai_gateway.guardrails.output).
+     - Configuration for output guardrail filters. See [_](#model_serving_endpointsnameai_gatewayguardrailsoutput).
   
   
-### model_serving_endpoints.<name>.ai_gateway.guardrails.input
+### model_serving_endpoints.\<name\>.ai_gateway.guardrails.input
   
 **`Type: Map`**
   
@@ -5201,7 +5205,7 @@ Configuration for input guardrail filters.
   
    * - `pii`
      - Map
-     - Configuration for guardrail PII filter. See [_](#model_serving_endpoints.<name>.ai_gateway.guardrails.input.pii).
+     - Configuration for guardrail PII filter. See [_](#model_serving_endpointsnameai_gatewayguardrailsinputpii).
   
    * - `safety`
      - Boolean
@@ -5212,7 +5216,7 @@ Configuration for input guardrail filters.
      - The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics.
   
   
-### model_serving_endpoints.<name>.ai_gateway.guardrails.input.pii
+### model_serving_endpoints.\<name\>.ai_gateway.guardrails.input.pii
   
 **`Type: Map`**
   
@@ -5232,7 +5236,7 @@ Configuration for guardrail PII filter.
      - Configuration for input guardrail filters.
   
   
-### model_serving_endpoints.<name>.ai_gateway.guardrails.output
+### model_serving_endpoints.\<name\>.ai_gateway.guardrails.output
   
 **`Type: Map`**
   
@@ -5253,7 +5257,7 @@ Configuration for output guardrail filters.
   
    * - `pii`
      - Map
-     - Configuration for guardrail PII filter. See [_](#model_serving_endpoints.<name>.ai_gateway.guardrails.output.pii).
+     - Configuration for guardrail PII filter. See [_](#model_serving_endpointsnameai_gatewayguardrailsoutputpii).
   
    * - `safety`
      - Boolean
@@ -5264,7 +5268,7 @@ Configuration for output guardrail filters.
      - The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics.
   
   
-### model_serving_endpoints.<name>.ai_gateway.guardrails.output.pii
+### model_serving_endpoints.\<name\>.ai_gateway.guardrails.output.pii
   
 **`Type: Map`**
   
@@ -5284,7 +5288,7 @@ Configuration for guardrail PII filter.
      - Configuration for input guardrail filters.
   
   
-### model_serving_endpoints.<name>.ai_gateway.inference_table_config
+### model_serving_endpoints.\<name\>.ai_gateway.inference_table_config
   
 **`Type: Map`**
   
@@ -5317,7 +5321,7 @@ Use these tables to monitor and audit data being sent to and received from model
      - The prefix of the table in Unity Catalog. NOTE: On update, you have to disable inference table first in order to change the prefix name.
   
   
-### model_serving_endpoints.<name>.ai_gateway.rate_limits
+### model_serving_endpoints.\<name\>.ai_gateway.rate_limits
   
 **`Type: Sequence`**
   
@@ -5345,7 +5349,7 @@ Configuration for rate limits which can be set to limit endpoint traffic.
      - Renewal period field for a rate limit. Currently, only 'minute' is supported.
   
   
-### model_serving_endpoints.<name>.ai_gateway.usage_tracking_config
+### model_serving_endpoints.\<name\>.ai_gateway.usage_tracking_config
   
 **`Type: Map`**
   
@@ -5366,7 +5370,7 @@ These tables allow you to monitor operational usage on endpoints and their assoc
      - Whether to enable usage tracking.
   
   
-### model_serving_endpoints.<name>.config
+### model_serving_endpoints.\<name\>.config
   
 **`Type: Map`**
   
@@ -5383,22 +5387,22 @@ The core config of the serving endpoint.
   
    * - `auto_capture_config`
      - Map
-     - Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog. Note: this field is deprecated for creating new provisioned throughput endpoints, or updating existing provisioned throughput endpoints that never have inference table configured; in these cases please use AI Gateway to manage inference tables. See [_](#model_serving_endpoints.<name>.config.auto_capture_config).
+     - Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog. Note: this field is deprecated for creating new provisioned throughput endpoints, or updating existing provisioned throughput endpoints that never have inference table configured; in these cases please use AI Gateway to manage inference tables. See [_](#model_serving_endpointsnameconfigauto_capture_config).
   
    * - `served_entities`
      - Sequence
-     - The list of served entities under the serving endpoint config. See [_](#model_serving_endpoints.<name>.config.served_entities).
+     - The list of served entities under the serving endpoint config. See [_](#model_serving_endpointsnameconfigserved_entities).
   
    * - `served_models`
      - Sequence
-     - (Deprecated, use served_entities instead) The list of served models under the serving endpoint config. See [_](#model_serving_endpoints.<name>.config.served_models).
+     - (Deprecated, use served_entities instead) The list of served models under the serving endpoint config. See [_](#model_serving_endpointsnameconfigserved_models).
   
    * - `traffic_config`
      - Map
-     - The traffic configuration associated with the serving endpoint config. See [_](#model_serving_endpoints.<name>.config.traffic_config).
+     - The traffic configuration associated with the serving endpoint config. See [_](#model_serving_endpointsnameconfigtraffic_config).
   
   
-### model_serving_endpoints.<name>.config.auto_capture_config
+### model_serving_endpoints.\<name\>.config.auto_capture_config
   
 **`Type: Map`**
   
@@ -5433,7 +5437,7 @@ in these cases please use AI Gateway to manage inference tables.
      - The prefix of the table in Unity Catalog. NOTE: On update, you cannot change the prefix name if the inference table is already enabled.
   
   
-### model_serving_endpoints.<name>.config.served_entities
+### model_serving_endpoints.\<name\>.config.served_entities
   
 **`Type: Sequence`**
   
@@ -5462,7 +5466,7 @@ The list of served entities under the serving endpoint config.
   
    * - `external_model`
      - Map
-     - The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same. See [_](#model_serving_endpoints.<name>.config.served_entities.external_model).
+     - The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same. See [_](#model_serving_endpointsnameconfigserved_entitiesexternal_model).
   
    * - `instance_profile_arn`
      - String
@@ -5493,7 +5497,7 @@ The list of served entities under the serving endpoint config.
      - The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).
   
   
-### model_serving_endpoints.<name>.config.served_entities.external_model
+### model_serving_endpoints.\<name\>.config.served_entities.external_model
   
 **`Type: Map`**
   
@@ -5510,27 +5514,27 @@ The external model to be served. NOTE: Only one of external_model and (entity_na
   
    * - `ai21labs_config`
      - Map
-     - AI21Labs Config. Only required if the provider is 'ai21labs'. See [_](#model_serving_endpoints.<name>.config.served_entities.external_model.ai21labs_config).
+     - AI21Labs Config. Only required if the provider is 'ai21labs'. See [_](#model_serving_endpointsnameconfigserved_entitiesexternal_modelai21labs_config).
   
    * - `amazon_bedrock_config`
      - Map
-     - Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'. See [_](#model_serving_endpoints.<name>.config.served_entities.external_model.amazon_bedrock_config).
+     - Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'. See [_](#model_serving_endpointsnameconfigserved_entitiesexternal_modelamazon_bedrock_config).
   
    * - `anthropic_config`
      - Map
-     - Anthropic Config. Only required if the provider is 'anthropic'. See [_](#model_serving_endpoints.<name>.config.served_entities.external_model.anthropic_config).
+     - Anthropic Config. Only required if the provider is 'anthropic'. See [_](#model_serving_endpointsnameconfigserved_entitiesexternal_modelanthropic_config).
   
    * - `cohere_config`
      - Map
-     - Cohere Config. Only required if the provider is 'cohere'. See [_](#model_serving_endpoints.<name>.config.served_entities.external_model.cohere_config).
+     - Cohere Config. Only required if the provider is 'cohere'. See [_](#model_serving_endpointsnameconfigserved_entitiesexternal_modelcohere_config).
   
    * - `databricks_model_serving_config`
      - Map
-     - Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'. See [_](#model_serving_endpoints.<name>.config.served_entities.external_model.databricks_model_serving_config).
+     - Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'. See [_](#model_serving_endpointsnameconfigserved_entitiesexternal_modeldatabricks_model_serving_config).
   
    * - `google_cloud_vertex_ai_config`
      - Map
-     - Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'. See [_](#model_serving_endpoints.<name>.config.served_entities.external_model.google_cloud_vertex_ai_config).
+     - Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'. See [_](#model_serving_endpointsnameconfigserved_entitiesexternal_modelgoogle_cloud_vertex_ai_config).
   
    * - `name`
      - String
@@ -5538,11 +5542,11 @@ The external model to be served. NOTE: Only one of external_model and (entity_na
   
    * - `openai_config`
      - Map
-     - OpenAI Config. Only required if the provider is 'openai'. See [_](#model_serving_endpoints.<name>.config.served_entities.external_model.openai_config).
+     - OpenAI Config. Only required if the provider is 'openai'. See [_](#model_serving_endpointsnameconfigserved_entitiesexternal_modelopenai_config).
   
    * - `palm_config`
      - Map
-     - PaLM Config. Only required if the provider is 'palm'. See [_](#model_serving_endpoints.<name>.config.served_entities.external_model.palm_config).
+     - PaLM Config. Only required if the provider is 'palm'. See [_](#model_serving_endpointsnameconfigserved_entitiesexternal_modelpalm_config).
   
    * - `provider`
      - String
@@ -5553,7 +5557,7 @@ The external model to be served. NOTE: Only one of external_model and (entity_na
      - The task type of the external model.
   
   
-### model_serving_endpoints.<name>.config.served_entities.external_model.ai21labs_config
+### model_serving_endpoints.\<name\>.config.served_entities.external_model.ai21labs_config
   
 **`Type: Map`**
   
@@ -5577,7 +5581,7 @@ AI21Labs Config. Only required if the provider is 'ai21labs'.
      - An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`.
   
   
-### model_serving_endpoints.<name>.config.served_entities.external_model.amazon_bedrock_config
+### model_serving_endpoints.\<name\>.config.served_entities.external_model.amazon_bedrock_config
   
 **`Type: Map`**
   
@@ -5617,7 +5621,7 @@ Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'.
      - The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.
   
   
-### model_serving_endpoints.<name>.config.served_entities.external_model.anthropic_config
+### model_serving_endpoints.\<name\>.config.served_entities.external_model.anthropic_config
   
 **`Type: Map`**
   
@@ -5641,7 +5645,7 @@ Anthropic Config. Only required if the provider is 'anthropic'.
      - The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.
   
   
-### model_serving_endpoints.<name>.config.served_entities.external_model.cohere_config
+### model_serving_endpoints.\<name\>.config.served_entities.external_model.cohere_config
   
 **`Type: Map`**
   
@@ -5669,7 +5673,7 @@ Cohere Config. Only required if the provider is 'cohere'.
      - The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.
   
   
-### model_serving_endpoints.<name>.config.served_entities.external_model.databricks_model_serving_config
+### model_serving_endpoints.\<name\>.config.served_entities.external_model.databricks_model_serving_config
   
 **`Type: Map`**
   
@@ -5697,7 +5701,7 @@ Databricks Model Serving Config. Only required if the provider is 'databricks-mo
      - The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model.
   
   
-### model_serving_endpoints.<name>.config.served_entities.external_model.google_cloud_vertex_ai_config
+### model_serving_endpoints.\<name\>.config.served_entities.external_model.google_cloud_vertex_ai_config
   
 **`Type: Map`**
   
@@ -5729,7 +5733,7 @@ Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-ve
      - This is the region for the Google Cloud Vertex AI Service. See [supported regions] for more details. Some models are only available in specific regions.  [supported regions]: https://cloud.google.com/vertex-ai/docs/general/locations
   
   
-### model_serving_endpoints.<name>.config.served_entities.external_model.openai_config
+### model_serving_endpoints.\<name\>.config.served_entities.external_model.openai_config
   
 **`Type: Map`**
   
@@ -5789,7 +5793,7 @@ OpenAI Config. Only required if the provider is 'openai'.
      - This is an optional field to specify the organization in OpenAI or Azure OpenAI.
   
   
-### model_serving_endpoints.<name>.config.served_entities.external_model.palm_config
+### model_serving_endpoints.\<name\>.config.served_entities.external_model.palm_config
   
 **`Type: Map`**
   
@@ -5813,7 +5817,7 @@ PaLM Config. Only required if the provider is 'palm'.
      - The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`.
   
   
-### model_serving_endpoints.<name>.config.served_models
+### model_serving_endpoints.\<name\>.config.served_models
   
 **`Type: Sequence`**
   
@@ -5869,7 +5873,7 @@ PaLM Config. Only required if the provider is 'palm'.
      - The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).
   
   
-### model_serving_endpoints.<name>.config.traffic_config
+### model_serving_endpoints.\<name\>.config.traffic_config
   
 **`Type: Map`**
   
@@ -5886,10 +5890,10 @@ The traffic configuration associated with the serving endpoint config.
   
    * - `routes`
      - Sequence
-     - The list of routes that define traffic to each served entity. See [_](#model_serving_endpoints.<name>.config.traffic_config.routes).
+     - The list of routes that define traffic to each served entity. See [_](#model_serving_endpointsnameconfigtraffic_configroutes).
   
   
-### model_serving_endpoints.<name>.config.traffic_config.routes
+### model_serving_endpoints.\<name\>.config.traffic_config.routes
   
 **`Type: Sequence`**
   
@@ -5913,7 +5917,7 @@ The list of routes that define traffic to each served entity.
      - The percentage of endpoint traffic to send to this route. It must be an integer between 0 and 100 inclusive.
   
   
-### model_serving_endpoints.<name>.permissions
+### model_serving_endpoints.\<name\>.permissions
   
 **`Type: Sequence`**
   
@@ -5945,7 +5949,7 @@ The list of routes that define traffic to each served entity.
      - The name of the user that has the permission set in level.
   
   
-### model_serving_endpoints.<name>.rate_limits
+### model_serving_endpoints.\<name\>.rate_limits
   
 **`Type: Sequence`**
   
@@ -5973,7 +5977,7 @@ Rate limits to be applied to the serving endpoint. NOTE: this field is deprecate
      - Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported.
   
   
-### model_serving_endpoints.<name>.tags
+### model_serving_endpoints.\<name\>.tags
   
 **`Type: Sequence`**
   
@@ -6031,7 +6035,7 @@ models:
   
    * - `latest_versions`
      - Sequence
-     - Collection of latest model versions for each stage. Only contains models with current `READY` status. See [_](#models.<name>.latest_versions).
+     - Collection of latest model versions for each stage. Only contains models with current `READY` status. See [_](#modelsnamelatest_versions).
   
    * - `name`
      - String
@@ -6039,18 +6043,18 @@ models:
   
    * - `permissions`
      - Sequence
-     - See [_](#models.<name>.permissions).
+     - See [_](#modelsnamepermissions).
   
    * - `tags`
      - Sequence
-     - Tags: Additional metadata key-value pairs for this `registered_model`. See [_](#models.<name>.tags).
+     - Tags: Additional metadata key-value pairs for this `registered_model`. See [_](#modelsnametags).
   
    * - `user_id`
      - String
      - User that created this `registered_model`
   
   
-### models.<name>.latest_versions
+### models.\<name\>.latest_versions
   
 **`Type: Sequence`**
   
@@ -6108,7 +6112,7 @@ Only contains models with current `READY` status.
   
    * - `tags`
      - Sequence
-     - Tags: Additional metadata key-value pairs for this `model_version`. See [_](#models.<name>.latest_versions.tags).
+     - Tags: Additional metadata key-value pairs for this `model_version`. See [_](#modelsnamelatest_versionstags).
   
    * - `user_id`
      - String
@@ -6119,7 +6123,7 @@ Only contains models with current `READY` status.
      - Model's version number.
   
   
-### models.<name>.latest_versions.tags
+### models.\<name\>.latest_versions.tags
   
 **`Type: Sequence`**
   
@@ -6143,7 +6147,7 @@ Tags: Additional metadata key-value pairs for this `model_version`.
      - The tag value.
   
   
-### models.<name>.permissions
+### models.\<name\>.permissions
   
 **`Type: Sequence`**
   
@@ -6175,7 +6179,7 @@ Tags: Additional metadata key-value pairs for this `model_version`.
      - The name of the user that has the permission set in level.
   
   
-### models.<name>.tags
+### models.\<name\>.tags
   
 **`Type: Sequence`**
   
@@ -6219,6 +6223,10 @@ pipelines:
      - Type
      - Description
   
+   * - `allow_duplicate_names`
+     - Boolean
+     - If false, deployment will fail if name conflicts with that of another pipeline.
+  
    * - `budget_policy_id`
      - String
      - Budget policy of this pipeline.
@@ -6233,7 +6241,7 @@ pipelines:
   
    * - `clusters`
      - Sequence
-     - Cluster settings for this pipeline deployment. See [_](#pipelines.<name>.clusters).
+     - Cluster settings for this pipeline deployment. See [_](#pipelinesnameclusters).
   
    * - `configuration`
      - Map
@@ -6245,23 +6253,27 @@ pipelines:
   
    * - `deployment`
      - Map
-     - Deployment type of this pipeline. See [_](#pipelines.<name>.deployment).
+     - Deployment type of this pipeline. See [_](#pipelinesnamedeployment).
   
    * - `development`
      - Boolean
      - Whether the pipeline is in Development mode. Defaults to false.
   
+   * - `dry_run`
+     - Boolean
+     - 
+  
    * - `edition`
      - String
      - Pipeline product edition.
   
    * - `filters`
      - Map
-     - Filters on which Pipeline packages to include in the deployed graph. See [_](#pipelines.<name>.filters).
+     - Filters on which Pipeline packages to include in the deployed graph. See [_](#pipelinesnamefilters).
   
    * - `gateway_definition`
      - Map
-     - The definition of a gateway pipeline to support change data capture. See [_](#pipelines.<name>.gateway_definition).
+     - The definition of a gateway pipeline to support change data capture. See [_](#pipelinesnamegateway_definition).
   
    * - `id`
      - String
@@ -6269,11 +6281,11 @@ pipelines:
   
    * - `ingestion_definition`
      - Map
-     - The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings. See [_](#pipelines.<name>.ingestion_definition).
+     - The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings. See [_](#pipelinesnameingestion_definition).
   
    * - `libraries`
      - Sequence
-     - Libraries or code needed by this deployment. See [_](#pipelines.<name>.libraries).
+     - Libraries or code needed by this deployment. See [_](#pipelinesnamelibraries).
   
    * - `name`
      - String
@@ -6281,11 +6293,11 @@ pipelines:
   
    * - `notifications`
      - Sequence
-     - List of notification settings for this pipeline. See [_](#pipelines.<name>.notifications).
+     - List of notification settings for this pipeline. See [_](#pipelinesnamenotifications).
   
    * - `permissions`
      - Sequence
-     - See [_](#pipelines.<name>.permissions).
+     - See [_](#pipelinesnamepermissions).
   
    * - `photon`
      - Boolean
@@ -6293,7 +6305,11 @@ pipelines:
   
    * - `restart_window`
      - Map
-     - Restart window of this pipeline. See [_](#pipelines.<name>.restart_window).
+     - Restart window of this pipeline. See [_](#pipelinesnamerestart_window).
+  
+   * - `run_as`
+     - Map
+     - Write-only setting, available only in Create/Update calls. Specifies the user or service principal that the pipeline runs as. If not specified, the pipeline runs as the user who created the pipeline.  Only `user_name` or `service_principal_name` can be specified. If both are specified, an error is thrown. See [_](#pipelinesnamerun_as).
   
    * - `schema`
      - String
@@ -6313,7 +6329,7 @@ pipelines:
   
    * - `trigger`
      - Map
-     - Which pipeline trigger to use. Deprecated: Use `continuous` instead. See [_](#pipelines.<name>.trigger).
+     - Which pipeline trigger to use. Deprecated: Use `continuous` instead. See [_](#pipelinesnametrigger).
   
   
 **Example**
@@ -6338,7 +6354,7 @@ resources:
             path: ./pipeline.py
 ```
   
-### pipelines.<name>.clusters
+### pipelines.\<name\>.clusters
   
 **`Type: Sequence`**
   
@@ -6359,19 +6375,19 @@ Cluster settings for this pipeline deployment.
   
    * - `autoscale`
      - Map
-     - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#pipelines.<name>.clusters.autoscale).
+     - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#pipelinesnameclustersautoscale).
   
    * - `aws_attributes`
      - Map
-     - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#pipelines.<name>.clusters.aws_attributes).
+     - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#pipelinesnameclustersaws_attributes).
   
    * - `azure_attributes`
      - Map
-     - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#pipelines.<name>.clusters.azure_attributes).
+     - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#pipelinesnameclustersazure_attributes).
   
    * - `cluster_log_conf`
      - Map
-     - The configuration for delivering spark logs to a long-term storage destination. Only dbfs destinations are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. . See [_](#pipelines.<name>.clusters.cluster_log_conf).
+     - The configuration for delivering spark logs to a long-term storage destination. Only dbfs destinations are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. . See [_](#pipelinesnameclusterscluster_log_conf).
   
    * - `custom_tags`
      - Map
@@ -6391,11 +6407,11 @@ Cluster settings for this pipeline deployment.
   
    * - `gcp_attributes`
      - Map
-     - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#pipelines.<name>.clusters.gcp_attributes).
+     - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#pipelinesnameclustersgcp_attributes).
   
    * - `init_scripts`
      - Sequence
-     - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `<destination>/<cluster-ID>/init_scripts`. See [_](#pipelines.<name>.clusters.init_scripts).
+     - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `<destination>/<cluster-ID>/init_scripts`. See [_](#pipelinesnameclustersinit_scripts).
   
    * - `instance_pool_id`
      - String
@@ -6430,7 +6446,7 @@ Cluster settings for this pipeline deployment.
      - SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified.
   
   
-### pipelines.<name>.clusters.autoscale
+### pipelines.\<name\>.clusters.autoscale
   
 **`Type: Map`**
   
@@ -6459,7 +6475,7 @@ Note: autoscaling works best with DB runtime versions 3.0 or later.
      - Databricks Enhanced Autoscaling optimizes cluster utilization by automatically allocating cluster resources based on workload volume, with minimal impact to the data processing latency of your pipelines. Enhanced Autoscaling is available for `updates` clusters only. The legacy autoscaling feature is used for `maintenance` clusters. 
   
   
-### pipelines.<name>.clusters.aws_attributes
+### pipelines.\<name\>.clusters.aws_attributes
   
 **`Type: Map`**
   
@@ -6516,7 +6532,7 @@ If not specified at cluster creation, a set of default values will be used.
      - Identifier for the availability zone/datacenter in which the cluster resides. This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. This is an optional field at cluster creation, and if not specified, a default zone will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. The list of available zones as well as the default value can be found by using the `List Zones` method.
   
   
-### pipelines.<name>.clusters.azure_attributes
+### pipelines.\<name\>.clusters.azure_attributes
   
 **`Type: Map`**
   
@@ -6542,14 +6558,14 @@ If not specified at cluster creation, a set of default values will be used.
   
    * - `log_analytics_info`
      - Map
-     - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#pipelines.<name>.clusters.azure_attributes.log_analytics_info).
+     - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#pipelinesnameclustersazure_attributeslog_analytics_info).
   
    * - `spot_bid_max_price`
      - Any
      - The max bid price to be used for Azure spot instances. The Max price for the bid cannot be higher than the on-demand price of the instance. If not specified, the default value is -1, which specifies that the instance cannot be evicted on the basis of price, and only on the basis of availability. Further, the value should > 0 or -1.
   
   
-### pipelines.<name>.clusters.azure_attributes.log_analytics_info
+### pipelines.\<name\>.clusters.azure_attributes.log_analytics_info
   
 **`Type: Map`**
   
@@ -6573,7 +6589,7 @@ Defines values necessary to configure and run Azure Log Analytics agent
      - <needs content added>
   
   
-### pipelines.<name>.clusters.cluster_log_conf
+### pipelines.\<name\>.clusters.cluster_log_conf
   
 **`Type: Map`**
   
@@ -6595,14 +6611,14 @@ the destination of executor logs is `$destination/$clusterId/executor`.
   
    * - `dbfs`
      - Map
-     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#pipelines.<name>.clusters.cluster_log_conf.dbfs).
+     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#pipelinesnameclusterscluster_log_confdbfs).
   
    * - `s3`
      - Map
-     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#pipelines.<name>.clusters.cluster_log_conf.s3).
+     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#pipelinesnameclusterscluster_log_confs3).
   
   
-### pipelines.<name>.clusters.cluster_log_conf.dbfs
+### pipelines.\<name\>.clusters.cluster_log_conf.dbfs
   
 **`Type: Map`**
   
@@ -6623,7 +6639,7 @@ destination needs to be provided. e.g.
      - dbfs destination, e.g. `dbfs:/my/path`
   
   
-### pipelines.<name>.clusters.cluster_log_conf.s3
+### pipelines.\<name\>.clusters.cluster_log_conf.s3
   
 **`Type: Map`**
   
@@ -6670,7 +6686,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in
      - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
   
   
-### pipelines.<name>.clusters.gcp_attributes
+### pipelines.\<name\>.clusters.gcp_attributes
   
 **`Type: Map`**
   
@@ -6711,7 +6727,7 @@ If not specified at cluster creation, a set of default values will be used.
      - Identifier for the availability zone in which the cluster resides. This can be one of the following: - "HA" => High availability, spread nodes across availability zones for a Databricks deployment region [default] - "AUTO" => Databricks picks an availability zone to schedule the cluster on. - A GCP availability zone => Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones.
   
   
-### pipelines.<name>.clusters.init_scripts
+### pipelines.\<name\>.clusters.init_scripts
   
 **`Type: Sequence`**
   
@@ -6728,34 +6744,34 @@ The configuration for storing init scripts. Any number of destinations can be sp
   
    * - `abfss`
      - Map
-     - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>" } }. See [_](#pipelines.<name>.clusters.init_scripts.abfss).
+     - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>" } }. See [_](#pipelinesnameclustersinit_scriptsabfss).
   
    * - `dbfs`
      - Map
-     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#pipelines.<name>.clusters.init_scripts.dbfs).
+     - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#pipelinesnameclustersinit_scriptsdbfs).
   
    * - `file`
      - Map
-     - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#pipelines.<name>.clusters.init_scripts.file).
+     - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#pipelinesnameclustersinit_scriptsfile).
   
    * - `gcs`
      - Map
-     - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#pipelines.<name>.clusters.init_scripts.gcs).
+     - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#pipelinesnameclustersinit_scriptsgcs).
   
    * - `s3`
      - Map
-     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#pipelines.<name>.clusters.init_scripts.s3).
+     - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#pipelinesnameclustersinit_scriptss3).
   
    * - `volumes`
      - Map
-     - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#pipelines.<name>.clusters.init_scripts.volumes).
+     - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#pipelinesnameclustersinit_scriptsvolumes).
   
    * - `workspace`
      - Map
-     - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#pipelines.<name>.clusters.init_scripts.workspace).
+     - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#pipelinesnameclustersinit_scriptsworkspace).
   
   
-### pipelines.<name>.clusters.init_scripts.abfss
+### pipelines.\<name\>.clusters.init_scripts.abfss
   
 **`Type: Map`**
   
@@ -6776,7 +6792,7 @@ destination needs to be provided. e.g.
      - abfss destination, e.g. `abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>`.
   
   
-### pipelines.<name>.clusters.init_scripts.dbfs
+### pipelines.\<name\>.clusters.init_scripts.dbfs
   
 **`Type: Map`**
   
@@ -6797,7 +6813,7 @@ destination needs to be provided. e.g.
      - dbfs destination, e.g. `dbfs:/my/path`
   
   
-### pipelines.<name>.clusters.init_scripts.file
+### pipelines.\<name\>.clusters.init_scripts.file
   
 **`Type: Map`**
   
@@ -6818,7 +6834,7 @@ destination needs to be provided. e.g.
      - local file destination, e.g. `file:/my/local/file.sh`
   
   
-### pipelines.<name>.clusters.init_scripts.gcs
+### pipelines.\<name\>.clusters.init_scripts.gcs
   
 **`Type: Map`**
   
@@ -6839,7 +6855,7 @@ destination needs to be provided. e.g.
      - GCS destination/URI, e.g. `gs://my-bucket/some-prefix`
   
   
-### pipelines.<name>.clusters.init_scripts.s3
+### pipelines.\<name\>.clusters.init_scripts.s3
   
 **`Type: Map`**
   
@@ -6886,7 +6902,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in
      - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used.
   
   
-### pipelines.<name>.clusters.init_scripts.volumes
+### pipelines.\<name\>.clusters.init_scripts.volumes
   
 **`Type: Map`**
   
@@ -6907,7 +6923,7 @@ destination needs to be provided. e.g.
      - Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`
   
   
-### pipelines.<name>.clusters.init_scripts.workspace
+### pipelines.\<name\>.clusters.init_scripts.workspace
   
 **`Type: Map`**
   
@@ -6928,7 +6944,7 @@ destination needs to be provided. e.g.
      - workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`
   
   
-### pipelines.<name>.deployment
+### pipelines.\<name\>.deployment
   
 **`Type: Map`**
   
@@ -6952,7 +6968,7 @@ Deployment type of this pipeline.
      - The path to the file containing metadata about the deployment.
   
   
-### pipelines.<name>.filters
+### pipelines.\<name\>.filters
   
 **`Type: Map`**
   
@@ -6976,7 +6992,7 @@ Filters on which Pipeline packages to include in the deployed graph.
      - Paths to include.
   
   
-### pipelines.<name>.gateway_definition
+### pipelines.\<name\>.gateway_definition
   
 **`Type: Map`**
   
@@ -7012,7 +7028,7 @@ The definition of a gateway pipeline to support change data capture.
      - Required, Immutable. The name of the schema for the gateway pipelines's storage location.
   
   
-### pipelines.<name>.ingestion_definition
+### pipelines.\<name\>.ingestion_definition
   
 **`Type: Map`**
   
@@ -7037,14 +7053,14 @@ The configuration for a managed ingestion pipeline. These settings cannot be use
   
    * - `objects`
      - Sequence
-     - Required. Settings specifying tables to replicate and the destination for the replicated tables. See [_](#pipelines.<name>.ingestion_definition.objects).
+     - Required. Settings specifying tables to replicate and the destination for the replicated tables. See [_](#pipelinesnameingestion_definitionobjects).
   
    * - `table_configuration`
      - Map
-     - Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline. See [_](#pipelines.<name>.ingestion_definition.table_configuration).
+     - Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline. See [_](#pipelinesnameingestion_definitiontable_configuration).
   
   
-### pipelines.<name>.ingestion_definition.objects
+### pipelines.\<name\>.ingestion_definition.objects
   
 **`Type: Sequence`**
   
@@ -7061,18 +7077,18 @@ Required. Settings specifying tables to replicate and the destination for the re
   
    * - `report`
      - Map
-     - Select a specific source report. See [_](#pipelines.<name>.ingestion_definition.objects.report).
+     - Select a specific source report. See [_](#pipelinesnameingestion_definitionobjectsreport).
   
    * - `schema`
      - Map
-     - Select all tables from a specific source schema. See [_](#pipelines.<name>.ingestion_definition.objects.schema).
+     - Select all tables from a specific source schema. See [_](#pipelinesnameingestion_definitionobjectsschema).
   
    * - `table`
      - Map
-     - Select a specific source table. See [_](#pipelines.<name>.ingestion_definition.objects.table).
+     - Select a specific source table. See [_](#pipelinesnameingestion_definitionobjectstable).
   
   
-### pipelines.<name>.ingestion_definition.objects.report
+### pipelines.\<name\>.ingestion_definition.objects.report
   
 **`Type: Map`**
   
@@ -7105,10 +7121,10 @@ Select a specific source report.
   
    * - `table_configuration`
      - Map
-     - Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object. See [_](#pipelines.<name>.ingestion_definition.objects.report.table_configuration).
+     - Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object. See [_](#pipelinesnameingestion_definitionobjectsreporttable_configuration).
   
   
-### pipelines.<name>.ingestion_definition.objects.report.table_configuration
+### pipelines.\<name\>.ingestion_definition.objects.report.table_configuration
   
 **`Type: Map`**
   
@@ -7140,7 +7156,7 @@ Configuration settings to control the ingestion of tables. These settings overri
      - The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order.
   
   
-### pipelines.<name>.ingestion_definition.objects.schema
+### pipelines.\<name\>.ingestion_definition.objects.schema
   
 **`Type: Map`**
   
@@ -7173,10 +7189,10 @@ Select all tables from a specific source schema.
   
    * - `table_configuration`
      - Map
-     - Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object. See [_](#pipelines.<name>.ingestion_definition.objects.schema.table_configuration).
+     - Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object. See [_](#pipelinesnameingestion_definitionobjectsschematable_configuration).
   
   
-### pipelines.<name>.ingestion_definition.objects.schema.table_configuration
+### pipelines.\<name\>.ingestion_definition.objects.schema.table_configuration
   
 **`Type: Map`**
   
@@ -7208,7 +7224,7 @@ Configuration settings to control the ingestion of tables. These settings are ap
      - The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order.
   
   
-### pipelines.<name>.ingestion_definition.objects.table
+### pipelines.\<name\>.ingestion_definition.objects.table
   
 **`Type: Map`**
   
@@ -7249,10 +7265,10 @@ Select a specific source table.
   
    * - `table_configuration`
      - Map
-     - Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec. See [_](#pipelines.<name>.ingestion_definition.objects.table.table_configuration).
+     - Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec. See [_](#pipelinesnameingestion_definitionobjectstabletable_configuration).
   
   
-### pipelines.<name>.ingestion_definition.objects.table.table_configuration
+### pipelines.\<name\>.ingestion_definition.objects.table.table_configuration
   
 **`Type: Map`**
   
@@ -7284,7 +7300,7 @@ Configuration settings to control the ingestion of tables. These settings overri
      - The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order.
   
   
-### pipelines.<name>.ingestion_definition.table_configuration
+### pipelines.\<name\>.ingestion_definition.table_configuration
   
 **`Type: Map`**
   
@@ -7316,7 +7332,7 @@ Configuration settings to control the ingestion of tables. These settings are ap
      - The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order.
   
   
-### pipelines.<name>.libraries
+### pipelines.\<name\>.libraries
   
 **`Type: Sequence`**
   
@@ -7333,7 +7349,7 @@ Libraries or code needed by this deployment.
   
    * - `file`
      - Map
-     - The path to a file that defines a pipeline and is stored in the Databricks Repos. . See [_](#pipelines.<name>.libraries.file).
+     - The path to a file that defines a pipeline and is stored in the Databricks Repos. . See [_](#pipelinesnamelibrariesfile).
   
    * - `jar`
      - String
@@ -7341,18 +7357,18 @@ Libraries or code needed by this deployment.
   
    * - `maven`
      - Map
-     - Specification of a maven library to be installed. . See [_](#pipelines.<name>.libraries.maven).
+     - Specification of a maven library to be installed. . See [_](#pipelinesnamelibrariesmaven).
   
    * - `notebook`
      - Map
-     - The path to a notebook that defines a pipeline and is stored in the Databricks workspace. . See [_](#pipelines.<name>.libraries.notebook).
+     - The path to a notebook that defines a pipeline and is stored in the Databricks workspace. . See [_](#pipelinesnamelibrariesnotebook).
   
    * - `whl`
      - String
      - URI of the whl to be installed.
   
   
-### pipelines.<name>.libraries.file
+### pipelines.\<name\>.libraries.file
   
 **`Type: Map`**
   
@@ -7373,7 +7389,7 @@ The path to a file that defines a pipeline and is stored in the Databricks Repos
      - The absolute path of the file.
   
   
-### pipelines.<name>.libraries.maven
+### pipelines.\<name\>.libraries.maven
   
 **`Type: Map`**
   
@@ -7402,7 +7418,7 @@ Specification of a maven library to be installed.
      - Maven repo to install the Maven package from. If omitted, both Maven Central Repository and Spark Packages are searched.
   
   
-### pipelines.<name>.libraries.notebook
+### pipelines.\<name\>.libraries.notebook
   
 **`Type: Map`**
   
@@ -7423,7 +7439,7 @@ The path to a notebook that defines a pipeline and is stored in the Databricks w
      - The absolute path of the notebook.
   
   
-### pipelines.<name>.notifications
+### pipelines.\<name\>.notifications
   
 **`Type: Sequence`**
   
@@ -7447,7 +7463,7 @@ List of notification settings for this pipeline.
      - A list of email addresses notified when a configured alert is triggered. 
   
   
-### pipelines.<name>.permissions
+### pipelines.\<name\>.permissions
   
 **`Type: Sequence`**
   
@@ -7479,7 +7495,7 @@ List of notification settings for this pipeline.
      - The name of the user that has the permission set in level.
   
   
-### pipelines.<name>.restart_window
+### pipelines.\<name\>.restart_window
   
 **`Type: Map`**
   
@@ -7507,7 +7523,7 @@ Restart window of this pipeline.
      - Time zone id of restart window. See https://docs.databricks.com/sql/language-manual/sql-ref-syntax-aux-conf-mgmt-set-timezone.html for details. If not specified, UTC will be used.
   
   
-### pipelines.<name>.restart_window.days_of_week
+### pipelines.\<name\>.restart_window.days_of_week
   
 **`Type: Sequence`**
   
@@ -7515,7 +7531,33 @@ Days of week in which the restart is allowed to happen (within a five-hour windo
 If not specified all days of the week will be used.
   
   
-### pipelines.<name>.trigger
+### pipelines.\<name\>.run_as
+  
+**`Type: Map`**
+  
+Write-only setting, available only in Create/Update calls. Specifies the user or service principal that the pipeline runs as. If not specified, the pipeline runs as the user who created the pipeline.
+
+Only `user_name` or `service_principal_name` can be specified. If both are specified, an error is thrown.
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `service_principal_name`
+     - String
+     - Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.
+  
+   * - `user_name`
+     - String
+     - The email of an active workspace user. Users can only set this field to their own email.
+  
+  
+### pipelines.\<name\>.trigger
   
 **`Type: Map`**
   
@@ -7532,14 +7574,14 @@ Which pipeline trigger to use. Deprecated: Use `continuous` instead.
   
    * - `cron`
      - Map
-     - See [_](#pipelines.<name>.trigger.cron).
+     - See [_](#pipelinesnametriggercron).
   
    * - `manual`
      - Map
      - 
   
   
-### pipelines.<name>.trigger.cron
+### pipelines.\<name\>.trigger.cron
   
 **`Type: Map`**
   
@@ -7563,7 +7605,7 @@ Which pipeline trigger to use. Deprecated: Use `continuous` instead.
      - 
   
   
-### pipelines.<name>.trigger.manual
+### pipelines.\<name\>.trigger.manual
   
 **`Type: Map`**
   
@@ -7600,19 +7642,19 @@ quality_monitors:
   
    * - `custom_metrics`
      - Sequence
-     - Custom metrics to compute on the monitored table. These can be aggregate metrics, derived metrics (from already computed aggregate metrics), or drift metrics (comparing metrics across time windows). . See [_](#quality_monitors.<name>.custom_metrics).
+     - Custom metrics to compute on the monitored table. These can be aggregate metrics, derived metrics (from already computed aggregate metrics), or drift metrics (comparing metrics across time windows). . See [_](#quality_monitorsnamecustom_metrics).
   
    * - `data_classification_config`
      - Map
-     - The data classification config for the monitor. See [_](#quality_monitors.<name>.data_classification_config).
+     - The data classification config for the monitor. See [_](#quality_monitorsnamedata_classification_config).
   
    * - `inference_log`
      - Map
-     - Configuration for monitoring inference logs. See [_](#quality_monitors.<name>.inference_log).
+     - Configuration for monitoring inference logs. See [_](#quality_monitorsnameinference_log).
   
    * - `notifications`
      - Map
-     - The notification settings for the monitor. See [_](#quality_monitors.<name>.notifications).
+     - The notification settings for the monitor. See [_](#quality_monitorsnamenotifications).
   
    * - `output_schema_name`
      - String
@@ -7620,7 +7662,7 @@ quality_monitors:
   
    * - `schedule`
      - Map
-     - The schedule for automatically updating and refreshing metric tables. See [_](#quality_monitors.<name>.schedule).
+     - The schedule for automatically updating and refreshing metric tables. See [_](#quality_monitorsnameschedule).
   
    * - `skip_builtin_dashboard`
      - Boolean
@@ -7640,7 +7682,7 @@ quality_monitors:
   
    * - `time_series`
      - Map
-     - Configuration for monitoring time series tables. See [_](#quality_monitors.<name>.time_series).
+     - Configuration for monitoring time series tables. See [_](#quality_monitorsnametime_series).
   
    * - `warehouse_id`
      - String
@@ -7670,7 +7712,7 @@ resources:
         timezone_id: UTC
 ```
   
-### quality_monitors.<name>.custom_metrics
+### quality_monitors.\<name\>.custom_metrics
   
 **`Type: Sequence`**
   
@@ -7709,7 +7751,7 @@ windows).
      - Can only be one of ``"CUSTOM_METRIC_TYPE_AGGREGATE"``, ``"CUSTOM_METRIC_TYPE_DERIVED"``, or ``"CUSTOM_METRIC_TYPE_DRIFT"``. The ``"CUSTOM_METRIC_TYPE_AGGREGATE"`` and ``"CUSTOM_METRIC_TYPE_DERIVED"`` metrics are computed on a single table, whereas the ``"CUSTOM_METRIC_TYPE_DRIFT"`` compare metrics across baseline and input table, or across the two consecutive time windows. - CUSTOM_METRIC_TYPE_AGGREGATE: only depend on the existing columns in your table - CUSTOM_METRIC_TYPE_DERIVED: depend on previously computed aggregate metrics - CUSTOM_METRIC_TYPE_DRIFT:  depend on previously computed aggregate or derived metrics 
   
   
-### quality_monitors.<name>.data_classification_config
+### quality_monitors.\<name\>.data_classification_config
   
 **`Type: Map`**
   
@@ -7729,7 +7771,7 @@ The data classification config for the monitor.
      - Whether data classification is enabled.
   
   
-### quality_monitors.<name>.inference_log
+### quality_monitors.\<name\>.inference_log
   
 **`Type: Map`**
   
@@ -7773,7 +7815,7 @@ Configuration for monitoring inference logs.
      - Column that contains the timestamps of requests. The column must be one of the following: - A ``TimestampType`` column - A column whose values can be converted to timestamps through the pyspark   ``to_timestamp`` [function](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html). 
   
   
-### quality_monitors.<name>.notifications
+### quality_monitors.\<name\>.notifications
   
 **`Type: Map`**
   
@@ -7790,14 +7832,14 @@ The notification settings for the monitor.
   
    * - `on_failure`
      - Map
-     - Who to send notifications to on monitor failure. See [_](#quality_monitors.<name>.notifications.on_failure).
+     - Who to send notifications to on monitor failure. See [_](#quality_monitorsnamenotificationson_failure).
   
    * - `on_new_classification_tag_detected`
      - Map
-     - Who to send notifications to when new data classification tags are detected. See [_](#quality_monitors.<name>.notifications.on_new_classification_tag_detected).
+     - Who to send notifications to when new data classification tags are detected. See [_](#quality_monitorsnamenotificationson_new_classification_tag_detected).
   
   
-### quality_monitors.<name>.notifications.on_failure
+### quality_monitors.\<name\>.notifications.on_failure
   
 **`Type: Map`**
   
@@ -7817,7 +7859,7 @@ Who to send notifications to on monitor failure.
      - The list of email addresses to send the notification to. A maximum of 5 email addresses is supported.
   
   
-### quality_monitors.<name>.notifications.on_new_classification_tag_detected
+### quality_monitors.\<name\>.notifications.on_new_classification_tag_detected
   
 **`Type: Map`**
   
@@ -7837,7 +7879,7 @@ Who to send notifications to when new data classification tags are detected.
      - The list of email addresses to send the notification to. A maximum of 5 email addresses is supported.
   
   
-### quality_monitors.<name>.schedule
+### quality_monitors.\<name\>.schedule
   
 **`Type: Map`**
   
@@ -7865,14 +7907,14 @@ The schedule for automatically updating and refreshing metric tables.
      - The timezone id (e.g., ``"PST"``) in which to evaluate the quartz expression. 
   
   
-### quality_monitors.<name>.snapshot
+### quality_monitors.\<name\>.snapshot
   
 **`Type: Map`**
   
 Configuration for monitoring snapshot tables.
   
   
-### quality_monitors.<name>.time_series
+### quality_monitors.\<name\>.time_series
   
 **`Type: Map`**
   
@@ -7926,7 +7968,7 @@ registered_models:
   
    * - `grants`
      - Sequence
-     - See [_](#registered_models.<name>.grants).
+     - See [_](#registered_modelsnamegrants).
   
    * - `name`
      - String
@@ -7959,7 +8001,7 @@ resources:
             principal: account users
 ```
   
-### registered_models.<name>.grants
+### registered_models.\<name\>.grants
   
 **`Type: Sequence`**
   
@@ -8016,7 +8058,7 @@ schemas:
   
    * - `grants`
      - Sequence
-     - See [_](#schemas.<name>.grants).
+     - See [_](#schemasnamegrants).
   
    * - `name`
      - String
@@ -8073,7 +8115,7 @@ resources:
       catalog_name: main
   ```
   
-### schemas.<name>.grants
+### schemas.\<name\>.grants
   
 **`Type: Sequence`**
   
@@ -8131,7 +8173,7 @@ volumes:
   
    * - `grants`
      - Sequence
-     - See [_](#volumes.<name>.grants).
+     - See [_](#volumesnamegrants).
   
    * - `name`
      - String
@@ -8165,7 +8207,7 @@ resources:
 
 For an example bundle that runs a job that writes to a file in <UC> volume, see the [bundle-examples GitHub repository](https://github.com/databricks/bundle-examples/tree/main/knowledge_base/write_from_job_to_volume).
   
-### volumes.<name>.grants
+### volumes.\<name\>.grants
   
 **`Type: Sequence`**
   
diff --git a/bundle/docsgen/testdata/anchors.md b/bundle/docsgen/testdata/anchors.md
new file mode 100644
index 000000000..0145d8cc9
--- /dev/null
+++ b/bundle/docsgen/testdata/anchors.md
@@ -0,0 +1,28 @@
+Header
+  
+## some_field
+  
+**`Type: Map`**
+  
+This is a description
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `my_attribute`
+     - Map
+     - Desc with link. See [_](#some_fieldnamemy_attribute).
+  
+  
+### some_field.\<name\>.my_attribute
+  
+**`Type: Boolean`**
+  
+Another description
+  
\ No newline at end of file
diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml
index 2d1a6a3d8..c10f43b04 100644
--- a/bundle/internal/schema/annotations.yml
+++ b/bundle/internal/schema/annotations.yml
@@ -414,16 +414,6 @@ github.com/databricks/cli/bundle/config/resources.Permission:
   "user_name":
     "description": |-
       The name of the user that has the permission set in level.
-github.com/databricks/cli/bundle/config/resources.Pipeline:
-  "allow_duplicate_names":
-    "description": |-
-      PLACEHOLDER
-  "dry_run":
-    "description": |-
-      PLACEHOLDER
-  "run_as":
-    "description": |-
-      PLACEHOLDER
 github.com/databricks/cli/bundle/config/variable.Lookup:
   "alert":
     "description": |-
diff --git a/bundle/internal/schema/main_test.go b/bundle/internal/schema/main_test.go
index 051243c4d..620f1cb70 100644
--- a/bundle/internal/schema/main_test.go
+++ b/bundle/internal/schema/main_test.go
@@ -124,3 +124,23 @@ func getAnnotations(path string) (annotation.File, error) {
 	err = yaml.Unmarshal(b, &data)
 	return data, err
 }
+
+func TestNoDuplicatedAnnotations(t *testing.T) {
+	// Check for duplicated annotations in annotation files
+	files := []string{
+		"annotations_openapi_overrides.yml",
+		"annotations.yml",
+	}
+
+	annotations := map[string]bool{}
+	for _, file := range files {
+		annotationsFile, err := getAnnotations(file)
+		assert.NoError(t, err)
+		for k := range annotationsFile {
+			if _, ok := annotations[k]; ok {
+				t.Errorf("Annotation `%s` is duplicated in %s", k, file)
+			}
+			annotations[k] = true
+		}
+	}
+}