Merge branch 'main' of github.com:databricks/cli into feat/config-reference-doc-autogen

2025-01-10 16:07:13 +01:00 · 2025-01-10 16:07:13 +01:00 · 954ef76fe9
parent bad77bd358 dc3a157fdc
commit 954ef76fe9
379 changed files with 6133 additions and 4481 deletions
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@ -0,0 +1 @@
+* @pietern @andrewnester @shreyas-goenka @denik
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@ -4,3 +4,7 @@ updates:
    directory: "/"
    schedule:
      interval: "weekly"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"
--- a/.github/workflows/close-stale-issues.yml
+++ b/.github/workflows/close-stale-issues.yml
@ -0,0 +1,38 @@
+name: "Close Stale Issues"
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * *" # Run at midnight every day
+
+jobs:
+  cleanup:
+    name: Stale issue job
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
+    permissions:
+      issues: write
+      contents: read
+      pull-requests: write
+
+    steps:
+      - uses: actions/stale@v9
+        with:
+          stale-issue-message: This issue has not received a response in a while. If you want to keep this issue open, please leave a comment below and auto-close will be canceled.
+          stale-pr-message: This PR has not received an update in a while. If you want to keep this PR open, please leave a comment below or push a new commit and auto-close will be canceled.
+
+          # These labels are required
+          stale-issue-label: Stale
+          stale-pr-label: Stale
+
+          exempt-issue-labels: No Autoclose
+          exempt-pr-labels: No Autoclose
+
+          # Issue timing
+          days-before-stale: 60
+          days-before-close: 30
+
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          loglevel: DEBUG
--- a/.github/workflows/external-message.yml
+++ b/.github/workflows/external-message.yml
@ -13,10 +13,17 @@ on:

 jobs:
  comment-on-pr:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
    permissions:
      pull-requests: write

+    # Only run this job for PRs from forks.
+    # Integration tests are not run automatically for PRs from forks.
+    if: "${{ github.event.pull_request.head.repo.fork }}"
+
    steps:
      - uses: actions/checkout@v4

@ -43,7 +50,7 @@ jobs:
        run: |
          gh pr comment ${{ github.event.pull_request.number }} --body \
          "<!-- INTEGRATION_TESTS_MANUAL -->
-          If integration tests don't run automatically, an authorized user can run them manually by following the instructions below:
+          An authorized user can trigger integration tests manually by following the instructions below:

          Trigger:
          [go/deco-tests-run/cli](https://go/deco-tests-run/cli)
--- a/.github/workflows/integration-approve.yml
+++ b/.github/workflows/integration-approve.yml
@ -17,7 +17,9 @@ jobs:
  #   * Avoid running integration tests twice, since it was already run at the tip of the branch before squashing.
  #
  trigger:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco

    steps:
      - name: Auto-approve squashed commit
--- a/.github/workflows/integration-main.yml
+++ b/.github/workflows/integration-main.yml
@ -11,7 +11,10 @@ jobs:
  # This workflow triggers the integration test workflow in a different repository.
  # It requires secrets from the "test-trigger-is" environment, which are only available to authorized users.
  trigger:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
    environment: "test-trigger-is"

    steps:
--- a/.github/workflows/integration-pr.yml
+++ b/.github/workflows/integration-pr.yml
@ -5,36 +5,20 @@ on:
    types: [opened, synchronize]

 jobs:
-  check-token:
-    runs-on: ubuntu-latest
-    environment: "test-trigger-is"
-
-    outputs:
-      has_token: ${{ steps.set-token-status.outputs.has_token }}
-
-    steps:
-      - name: Check if DECO_WORKFLOW_TRIGGER_APP_ID is set
-        id: set-token-status
-        run: |
-          if [ -z "${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}" ]; then
-            echo "DECO_WORKFLOW_TRIGGER_APP_ID is empty. User has no access to secrets."
-            echo "::set-output name=has_token::false"
-          else
-            echo "DECO_WORKFLOW_TRIGGER_APP_ID is set. User has access to secrets."
-            echo "::set-output name=has_token::true"
-          fi
-
  # Trigger for pull requests.
  #
  # This workflow triggers the integration test workflow in a different repository.
  # It requires secrets from the "test-trigger-is" environment, which are only available to authorized users.
-  # It depends on the "check-token" workflow to confirm access to this environment to avoid failures.
  trigger:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
    environment: "test-trigger-is"

-    if: needs.check-token.outputs.has_token == 'true'
-    needs: check-token
+    # Only run this job for PRs from branches on the main repository and not from forks.
+    # Workflows triggered by PRs from forks don't have access to the "test-trigger-is" environment.
+    if: "${{ !github.event.pull_request.head.repo.fork }}"

    steps:
      - name: Generate GitHub App Token
--- a/.github/workflows/publish-winget.yml
+++ b/.github/workflows/publish-winget.yml
@ -2,11 +2,19 @@ name: publish-winget

 on:
  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Tag to publish'
+        default: ''

 jobs:
  publish-to-winget-pkgs:
-    runs-on: windows-latest
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: windows-server-latest
+
    environment: release
+
    steps:
      - uses: vedantmgoyal2009/winget-releaser@93fd8b606a1672ec3e5c6c3bb19426be68d1a8b0 # https://github.com/vedantmgoyal2009/winget-releaser/releases/tag/v2
        with:
@ -14,3 +22,7 @@ jobs:
          installers-regex: 'windows_.*-signed\.zip$' # Only signed Windows releases
          token: ${{ secrets.ENG_DEV_ECOSYSTEM_BOT_TOKEN }}
          fork-user: eng-dev-ecosystem-bot
+
+          # Use the tag from the input, or the ref name if the input is not provided.
+          # The ref name is equal to the tag name when this workflow is triggered by the "sign-cli" command.
+          release-tag: ${{ inputs.tag || github.ref_name }}
--- a/.github/workflows/push.yml
+++ b/.github/workflows/push.yml
@ -13,9 +13,26 @@ on:
    # seed the build cache.
    branches:
      - main
+  schedule:
+    - cron: '0 0,12 * * *'  # Runs at 00:00 and 12:00 UTC daily
+
+env:
+  GOTESTSUM_FORMAT: github-actions

 jobs:
+  cleanups:
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+    steps:
+      - name: Clean up cache if running on schedule
+        if: ${{ github.event_name == 'schedule' }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: gh cache delete --all --repo databricks/cli || true
+
  tests:
+    needs: cleanups
    runs-on: ${{ matrix.os }}

    strategy:
@ -55,9 +72,10 @@ jobs:
          pip3 install wheel

      - name: Run tests
-        run: make testonly
+        run: make test

  golangci:
+    needs: cleanups
    name: lint
    runs-on: ubuntu-latest
    steps:
@ -65,6 +83,10 @@ jobs:
      - uses: actions/setup-go@v5
        with:
          go-version: 1.23.4
+          # Use different schema from regular job, to avoid overwriting the same key
+          cache-dependency-path: |
+            go.sum
+            .golangci.yaml
      - name: Run go mod tidy
        run: |
          go mod tidy
@ -75,10 +97,11 @@ jobs:
      - name: golangci-lint
        uses: golangci/golangci-lint-action@v6
        with:
-          version: v1.62.2
+          version: v1.63.4
          args: --timeout=15m

  validate-bundle-schema:
+    needs: cleanups
    runs-on: ubuntu-latest

    steps:
@ -89,6 +112,17 @@ jobs:
        uses: actions/setup-go@v5
        with:
          go-version: 1.23.4
+          # Use different schema from regular job, to avoid overwriting the same key
+          cache-dependency-path: |
+            go.sum
+            bundle/internal/schema/*.*
+
+      - name: Verify that the schema is up to date
+        run: |
+          if ! ( make schema && git diff --exit-code ); then
+            echo "The schema is not up to date. Please run 'make schema' and commit the changes."
+            exit 1
+          fi

      # Github repo: https://github.com/ajv-validator/ajv-cli
      - name: Install ajv-cli
--- a/.github/workflows/release-snapshot.yml
+++ b/.github/workflows/release-snapshot.yml
@ -20,7 +20,10 @@ on:

 jobs:
  goreleaser:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
    steps:
      - name: Checkout repository and submodules
        uses: actions/checkout@v4
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@ -9,9 +9,13 @@ on:

 jobs:
  goreleaser:
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
    outputs:
      artifacts: ${{ steps.releaser.outputs.artifacts }}
-    runs-on: ubuntu-latest
+
    steps:
      - name: Checkout repository and submodules
        uses: actions/checkout@v4
@ -54,8 +58,12 @@ jobs:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  create-setup-cli-release-pr:
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
    needs: goreleaser
-    runs-on: ubuntu-latest
+
    steps:
      - name: Set VERSION variable from tag
        run: |
@ -78,8 +86,12 @@ jobs:
            });

  create-homebrew-tap-release-pr:
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
    needs: goreleaser
-    runs-on: ubuntu-latest
+
    steps:
      - name: Set VERSION variable from tag
        run: |
@ -115,8 +127,12 @@ jobs:
            });

  create-vscode-extension-update-pr:
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
    needs: goreleaser
-    runs-on: ubuntu-latest
+
    steps:
      - name: Set VERSION variable from tag
        run: |
--- a/.golangci.yaml
+++ b/.golangci.yaml
@ -11,6 +11,10 @@ linters:
    - gofmt
    - gofumpt
    - goimports
+    - testifylint
+    - intrange
+    - mirror
+    - perfsprint
 linters-settings:
  govet:
    enable-all: true
@ -32,7 +36,12 @@ linters-settings:
  gofumpt:
    module-path: github.com/databricks/cli
    extra-rules: true
-  #goimports:
-  #  local-prefixes: github.com/databricks/cli
+  testifylint:
+    enable-all: true
+    disable:
+      # good check, but we have too many assert.(No)?Errorf? so excluding for now
+      - require-error
 issues:
  exclude-dirs-use-default: false  # recommended by docs https://golangci-lint.run/usage/false-positives/
+  max-issues-per-linter: 1000
+  max-same-issues: 1000
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -1,5 +1,14 @@
 # Version changelog

+## [Release] Release v0.238.0
+
+Bundles:
+ * Fix finding Python within virtualenv on Windows ([#2034](https://github.com/databricks/cli/pull/2034)).
+ * Include missing field descriptions in JSON schema ([#2045](https://github.com/databricks/cli/pull/2045)).
+ * Add validation for volume referenced from `artifact_path` ([#2050](https://github.com/databricks/cli/pull/2050)).
+ * Handle `${workspace.file_path}` references in source-linked deployments ([#2046](https://github.com/databricks/cli/pull/2046)).
+ * Set the write bit for files written during template initialization ([#2068](https://github.com/databricks/cli/pull/2068)).
+
 ## [Release] Release v0.237.0

 Bundles:
--- a/51
+++ b/51
@ -1,43 +1,44 @@
 default: build

-lint: vendor
-	@echo "✓ Linting source code with https://golangci-lint.run/ (with --fix)..."
-	@golangci-lint run --fix ./...
+PACKAGES=./acceptance/... ./libs/... ./internal/... ./cmd/... ./bundle/... .

-lintcheck: vendor
-	@echo "✓ Linting source code with https://golangci-lint.run/ ..."
-	@golangci-lint run ./...
+GOTESTSUM_FORMAT ?= pkgname-and-test-fails

-test: lint testonly
+lint:
+	golangci-lint run --fix

-testonly:
-	@echo "✓ Running tests ..."
-	@gotestsum --format pkgname-and-test-fails --no-summary=skipped --raw-command go test -v -json -short -coverprofile=coverage.txt ./...
+lintcheck:
+	golangci-lint run ./...

-coverage: test
-	@echo "✓ Opening coverage for unit tests ..."
-	@go tool cover -html=coverage.txt
+# Note 'make lint' will do formatting as well. However, if there are compilation errors,
+# formatting/goimports will not be applied by 'make lint'. However, it will be applied by 'make fmt'.
+# If you need to ensure that formatting & imports are always fixed, do "make fmt lint"
+fmt:
+	golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix ./...
+
+test:
+	gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped -- ${PACKAGES}
+
+cover:
+	gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped -- -coverprofile=coverage.txt ${PACKAGES}
+
+showcover:
+	go tool cover -html=coverage.txt

 build: vendor
-	@echo "✓ Building source code with go build ..."
-	@go build -mod vendor
+	go build -mod vendor

 snapshot:
-	@echo "✓ Building dev snapshot"
-	@go build -o .databricks/databricks
+	go build -o .databricks/databricks

 vendor:
-	@echo "✓ Filling vendor folder with library code ..."
-	@go mod vendor
+	go mod vendor

 schema:
-	@echo "✓ Generating json-schema ..."
-	@go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json
+	go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json

 docs:
-	@echo "✓ Generating docs using ./bundle/internal/schema/annotations.yml file..."
-	@go run ./bundle/internal/docs ./bundle/internal/schema ./bundle/internal/docs
-	@echo "✓ Writing docs to ./bundle/internal/docs/docs.md"
+	go run ./bundle/internal/docs ./bundle/internal/schema ./bundle/internal/docs

 INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./integration/..." -- -parallel 4 -timeout=2h

@ -47,4 +48,4 @@ integration:
 integration-short:
 	$(INTEGRATION) -short

-.PHONY: lint lintcheck test testonly coverage build snapshot vendor schema integration integration-short docs
+.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short docs
--- a/acceptance/README.md
+++ b/acceptance/README.md
@ -0,0 +1,19 @@
+Acceptance tests are blackbox tests that are run against compiled binary.
+
+Currently these tests are run against "fake" HTTP server pretending to be Databricks API. However, they will be extended to run against real environment as regular integration tests.
+
+To author a test,
+ - Add a new directory under `acceptance`. Any level of nesting is supported.
+ - Add `databricks.yml` there.
+ - Add `script` with commands to run, e.g. `$CLI bundle validate`. The test case is recognized by presence of `script`.
+
+The test runner will run script and capture output and compare it with `output.txt` file in the same directory.
+
+In order to write `output.txt` for the first time or overwrite it with the current output pass -update flag to go test.
+
+The scripts are run with `bash -e` so any errors will be propagated. They are captured in `output.txt` by appending `Exit code: N` line at the end.
+
+For more complex tests one can also use:
+- `errcode` helper: if the command fails with non-zero code, it appends `Exit code: N` to the output but returns success to caller (bash), allowing continuation of script.
+- `trace` helper: prints the arguments before executing the command.
+- custom output files: redirect output to custom file (it must start with `out`), e.g. `$CLI bundle validate > out.txt 2> out.error.txt`.
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@ -0,0 +1,311 @@
+package acceptance_test
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"slices"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/databricks/cli/internal/testutil"
+	"github.com/databricks/cli/libs/env"
+	"github.com/databricks/cli/libs/testdiff"
+	"github.com/stretchr/testify/require"
+)
+
+var KeepTmp = os.Getenv("KEEP_TMP") != ""
+
+const (
+	EntryPointScript = "script"
+	CleanupScript    = "script.cleanup"
+	PrepareScript    = "script.prepare"
+)
+
+var Scripts = map[string]bool{
+	EntryPointScript: true,
+	CleanupScript:    true,
+	PrepareScript:    true,
+}
+
+func TestAccept(t *testing.T) {
+	cwd, err := os.Getwd()
+	require.NoError(t, err)
+
+	execPath := BuildCLI(t, cwd)
+	// $CLI is what test scripts are using
+	t.Setenv("CLI", execPath)
+
+	// Make helper scripts available
+	t.Setenv("PATH", fmt.Sprintf("%s%c%s", filepath.Join(cwd, "bin"), os.PathListSeparator, os.Getenv("PATH")))
+
+	server := StartServer(t)
+	AddHandlers(server)
+	// Redirect API access to local server:
+	t.Setenv("DATABRICKS_HOST", fmt.Sprintf("http://127.0.0.1:%d", server.Port))
+	t.Setenv("DATABRICKS_TOKEN", "dapi1234")
+
+	homeDir := t.TempDir()
+	// Do not read user's ~/.databrickscfg
+	t.Setenv(env.HomeEnvVar(), homeDir)
+
+	repls := testdiff.ReplacementsContext{}
+	repls.Set(execPath, "$CLI")
+
+	testDirs := getTests(t)
+	require.NotEmpty(t, testDirs)
+	for _, dir := range testDirs {
+		t.Run(dir, func(t *testing.T) {
+			t.Parallel()
+			runTest(t, dir, repls)
+		})
+	}
+}
+
+func getTests(t *testing.T) []string {
+	testDirs := make([]string, 0, 128)
+
+	err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		name := filepath.Base(path)
+		if name == EntryPointScript {
+			// Presence of 'script' marks a test case in this directory
+			testDirs = append(testDirs, filepath.Dir(path))
+		}
+		return nil
+	})
+	require.NoError(t, err)
+
+	sort.Strings(testDirs)
+	return testDirs
+}
+
+func runTest(t *testing.T, dir string, repls testdiff.ReplacementsContext) {
+	var tmpDir string
+	var err error
+	if KeepTmp {
+		tempDirBase := filepath.Join(os.TempDir(), "acceptance")
+		_ = os.Mkdir(tempDirBase, 0o755)
+		tmpDir, err = os.MkdirTemp(tempDirBase, "")
+		require.NoError(t, err)
+		t.Logf("Created directory: %s", tmpDir)
+	} else {
+		tmpDir = t.TempDir()
+	}
+
+	scriptContents := readMergedScriptContents(t, dir)
+	testutil.WriteFile(t, filepath.Join(tmpDir, EntryPointScript), scriptContents)
+
+	inputs := make(map[string]bool, 2)
+	outputs := make(map[string]bool, 2)
+	err = CopyDir(dir, tmpDir, inputs, outputs)
+	require.NoError(t, err)
+
+	args := []string{"bash", "-euo", "pipefail", EntryPointScript}
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Dir = tmpDir
+	outB, err := cmd.CombinedOutput()
+
+	out := formatOutput(string(outB), err)
+	out = repls.Replace(out)
+	doComparison(t, filepath.Join(dir, "output.txt"), "script output", out)
+
+	for key := range outputs {
+		if key == "output.txt" {
+			// handled above
+			continue
+		}
+		pathNew := filepath.Join(tmpDir, key)
+		newValBytes, err := os.ReadFile(pathNew)
+		if err != nil {
+			if errors.Is(err, os.ErrNotExist) {
+				t.Errorf("%s: expected to find this file but could not (%s)", key, tmpDir)
+			} else {
+				t.Errorf("%s: could not read: %s", key, err)
+			}
+			continue
+		}
+		pathExpected := filepath.Join(dir, key)
+		newVal := repls.Replace(string(newValBytes))
+		doComparison(t, pathExpected, pathNew, newVal)
+	}
+
+	// Make sure there are not unaccounted for new files
+	files, err := os.ReadDir(tmpDir)
+	require.NoError(t, err)
+
+	for _, f := range files {
+		name := f.Name()
+		if _, ok := inputs[name]; ok {
+			continue
+		}
+		if _, ok := outputs[name]; ok {
+			continue
+		}
+		t.Errorf("Unexpected output: %s", f)
+		if strings.HasPrefix(name, "out") {
+			// We have a new file starting with "out"
+			// Show the contents & support overwrite mode for it:
+			pathNew := filepath.Join(tmpDir, name)
+			newVal := testutil.ReadFile(t, pathNew)
+			newVal = repls.Replace(newVal)
+			doComparison(t, filepath.Join(dir, name), filepath.Join(tmpDir, name), newVal)
+		}
+	}
+}
+
+func doComparison(t *testing.T, pathExpected, pathNew, valueNew string) {
+	valueNew = testdiff.NormalizeNewlines(valueNew)
+	valueExpected := string(readIfExists(t, pathExpected))
+	valueExpected = testdiff.NormalizeNewlines(valueExpected)
+	testdiff.AssertEqualTexts(t, pathExpected, pathNew, valueExpected, valueNew)
+	if testdiff.OverwriteMode {
+		if valueNew != "" {
+			t.Logf("Overwriting: %s", pathExpected)
+			testutil.WriteFile(t, pathExpected, valueNew)
+		} else {
+			t.Logf("Removing: %s", pathExpected)
+			_ = os.Remove(pathExpected)
+		}
+	}
+}
+
+// Returns combined script.prepare (root) + script.prepare (parent) + ... + script + ... + script.cleanup (parent) + ...
+// Note, cleanups are not executed if main script fails; that's not a huge issue, since it runs it temp dir.
+func readMergedScriptContents(t *testing.T, dir string) string {
+	scriptContents := testutil.ReadFile(t, filepath.Join(dir, EntryPointScript))
+	prepares := []string{}
+	cleanups := []string{}
+
+	for {
+		x := readIfExists(t, filepath.Join(dir, CleanupScript))
+		if len(x) > 0 {
+			cleanups = append(cleanups, string(x))
+		}
+
+		x = readIfExists(t, filepath.Join(dir, PrepareScript))
+		if len(x) > 0 {
+			prepares = append(prepares, string(x))
+		}
+
+		if dir == "" || dir == "." {
+			break
+		}
+
+		dir = filepath.Dir(dir)
+		require.True(t, filepath.IsLocal(dir))
+	}
+
+	slices.Reverse(prepares)
+	prepares = append(prepares, scriptContents)
+	prepares = append(prepares, cleanups...)
+	return strings.Join(prepares, "\n")
+}
+
+func BuildCLI(t *testing.T, cwd string) string {
+	execPath := filepath.Join(cwd, "build", "databricks")
+	if runtime.GOOS == "windows" {
+		execPath += ".exe"
+	}
+
+	start := time.Now()
+	args := []string{"go", "build", "-mod", "vendor", "-o", execPath}
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Dir = ".."
+	out, err := cmd.CombinedOutput()
+	elapsed := time.Since(start)
+	t.Logf("%s took %s", args, elapsed)
+	require.NoError(t, err, "go build failed: %s: %s\n%s", args, err, out)
+	if len(out) > 0 {
+		t.Logf("go build output: %s: %s", args, out)
+	}
+
+	// Quick check + warm up cache:
+	cmd = exec.Command(execPath, "--version")
+	out, err = cmd.CombinedOutput()
+	require.NoError(t, err, "%s --version failed: %s\n%s", execPath, err, out)
+	return execPath
+}
+
+func copyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, in)
+	return err
+}
+
+func formatOutput(out string, err error) string {
+	if err == nil {
+		return out
+	}
+	if exiterr, ok := err.(*exec.ExitError); ok {
+		exitCode := exiterr.ExitCode()
+		out += fmt.Sprintf("\nExit code: %d\n", exitCode)
+	} else {
+		out += fmt.Sprintf("\nError: %s\n", err)
+	}
+	return out
+}
+
+func readIfExists(t *testing.T, path string) []byte {
+	data, err := os.ReadFile(path)
+	if err == nil {
+		return data
+	}
+
+	if !errors.Is(err, os.ErrNotExist) {
+		t.Fatalf("%s: %s", path, err)
+	}
+	return []byte{}
+}
+
+func CopyDir(src, dst string, inputs, outputs map[string]bool) error {
+	return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		name := info.Name()
+
+		relPath, err := filepath.Rel(src, path)
+		if err != nil {
+			return err
+		}
+
+		if strings.HasPrefix(name, "out") {
+			outputs[relPath] = true
+			return nil
+		} else {
+			inputs[relPath] = true
+		}
+
+		if _, ok := Scripts[name]; ok {
+			return nil
+		}
+
+		destPath := filepath.Join(dst, relPath)
+
+		if info.IsDir() {
+			return os.MkdirAll(destPath, info.Mode())
+		}
+
+		return copyFile(path, destPath)
+	})
+}
--- a/acceptance/bin/sort_blocks.py
+++ b/acceptance/bin/sort_blocks.py
@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+"""
+Helper to sort blocks in text file. A block is a set of lines separated from others by empty line.
+
+This is to workaround non-determinism in the output.
+"""
+import sys
+
+blocks = []
+
+for line in sys.stdin:
+    if not line.strip():
+        if blocks and blocks[-1]:
+            blocks.append('')
+        continue
+    if not blocks:
+        blocks.append('')
+    blocks[-1] += line
+
+blocks.sort()
+print("\n".join(blocks))
--- a/acceptance/build/.gitignore
+++ b/acceptance/build/.gitignore
@ -0,0 +1 @@
+databricks
--- a/acceptance/bundle/override/clusters/databricks.yml
+++ b/acceptance/bundle/override/clusters/databricks.yml
@ -1,9 +1,6 @@
 bundle:
  name: clusters

-workspace:
-  host: https://acme.cloud.databricks.com/
-
 resources:
  clusters:
    foo:
--- a/acceptance/bundle/override/clusters/output.txt
+++ b/acceptance/bundle/override/clusters/output.txt
@ -0,0 +1,33 @@
+
+>>> $CLI bundle validate -o json -t default
+{
+  "autoscale": {
+    "max_workers": 7,
+    "min_workers": 2
+  },
+  "cluster_name": "foo",
+  "custom_tags": {},
+  "node_type_id": "i3.xlarge",
+  "num_workers": 2,
+  "spark_conf": {
+    "spark.executor.memory": "2g"
+  },
+  "spark_version": "13.3.x-scala2.12"
+}
+
+>>> $CLI bundle validate -o json -t development
+{
+  "autoscale": {
+    "max_workers": 3,
+    "min_workers": 1
+  },
+  "cluster_name": "foo-override",
+  "custom_tags": {},
+  "node_type_id": "m5.xlarge",
+  "num_workers": 3,
+  "spark_conf": {
+    "spark.executor.memory": "4g",
+    "spark.executor.memory2": "4g"
+  },
+  "spark_version": "15.2.x-scala2.12"
+}
--- a/acceptance/bundle/override/clusters/script
+++ b/acceptance/bundle/override/clusters/script
@ -0,0 +1,2 @@
+trace $CLI bundle validate -o json -t default | jq .resources.clusters.foo
+trace $CLI bundle validate -o json -t development | jq .resources.clusters.foo
--- a/acceptance/bundle/override/job_cluster/databricks.yml
+++ b/acceptance/bundle/override/job_cluster/databricks.yml
@ -1,9 +1,6 @@
 bundle:
  name: override_job_cluster

-workspace:
-  host: https://acme.cloud.databricks.com/
-
 resources:
  jobs:
    foo:
--- a/acceptance/bundle/override/job_cluster/output.txt
+++ b/acceptance/bundle/override/job_cluster/output.txt
@ -0,0 +1,56 @@
+
+>>> $CLI bundle validate -o json -t development
+{
+  "foo": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "job_clusters": [
+      {
+        "job_cluster_key": "key",
+        "new_cluster": {
+          "node_type_id": "i3.xlarge",
+          "num_workers": 1,
+          "spark_version": "13.3.x-scala2.12"
+        }
+      }
+    ],
+    "name": "job",
+    "permissions": [],
+    "queue": {
+      "enabled": true
+    },
+    "tags": {}
+  }
+}
+
+>>> $CLI bundle validate -o json -t staging
+{
+  "foo": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "job_clusters": [
+      {
+        "job_cluster_key": "key",
+        "new_cluster": {
+          "node_type_id": "i3.2xlarge",
+          "num_workers": 4,
+          "spark_version": "13.3.x-scala2.12"
+        }
+      }
+    ],
+    "name": "job",
+    "permissions": [],
+    "queue": {
+      "enabled": true
+    },
+    "tags": {}
+  }
+}
--- a/acceptance/bundle/override/job_cluster/script
+++ b/acceptance/bundle/override/job_cluster/script
@ -0,0 +1,2 @@
+trace $CLI bundle validate -o json -t development | jq '.resources.jobs'
+trace $CLI bundle validate -o json -t staging | jq '.resources.jobs'
--- a/acceptance/bundle/override/job_cluster_var/databricks.yml
+++ b/acceptance/bundle/override/job_cluster_var/databricks.yml
@ -0,0 +1,37 @@
+bundle:
+  name: override_job_cluster
+
+variables:
+  mykey:
+    default: key
+
+resources:
+  jobs:
+    foo:
+      name: job
+      job_clusters:
+        - job_cluster_key: key
+          new_cluster:
+            spark_version: 13.3.x-scala2.12
+
+targets:
+  development:
+    resources:
+      jobs:
+        foo:
+          job_clusters:
+            # This does not work because merging is done before resolution
+            - job_cluster_key: "${var.mykey}"
+              new_cluster:
+                node_type_id: i3.xlarge
+                num_workers: 1
+
+  staging:
+    resources:
+      jobs:
+        foo:
+          job_clusters:
+            - job_cluster_key: "${var.mykey}"
+              new_cluster:
+                node_type_id: i3.2xlarge
+                num_workers: 4
--- a/acceptance/bundle/override/job_cluster_var/output.txt
+++ b/acceptance/bundle/override/job_cluster_var/output.txt
@ -0,0 +1,84 @@
+
+>>> $CLI bundle validate -o json -t development
+{
+  "foo": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "job_clusters": [
+      {
+        "job_cluster_key": "key",
+        "new_cluster": {
+          "spark_version": "13.3.x-scala2.12"
+        }
+      },
+      {
+        "job_cluster_key": "key",
+        "new_cluster": {
+          "node_type_id": "i3.xlarge",
+          "num_workers": 1
+        }
+      }
+    ],
+    "name": "job",
+    "permissions": [],
+    "queue": {
+      "enabled": true
+    },
+    "tags": {}
+  }
+}
+
+>>> $CLI bundle validate -t development
+Name: override_job_cluster
+Target: development
+Workspace:
+  User: tester@databricks.com
+  Path: /Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development
+
+Validation OK!
+
+>>> $CLI bundle validate -o json -t staging
+{
+  "foo": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "job_clusters": [
+      {
+        "job_cluster_key": "key",
+        "new_cluster": {
+          "spark_version": "13.3.x-scala2.12"
+        }
+      },
+      {
+        "job_cluster_key": "key",
+        "new_cluster": {
+          "node_type_id": "i3.2xlarge",
+          "num_workers": 4
+        }
+      }
+    ],
+    "name": "job",
+    "permissions": [],
+    "queue": {
+      "enabled": true
+    },
+    "tags": {}
+  }
+}
+
+>>> $CLI bundle validate -t staging
+Name: override_job_cluster
+Target: staging
+Workspace:
+  User: tester@databricks.com
+  Path: /Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging
+
+Validation OK!
--- a/acceptance/bundle/override/job_cluster_var/script
+++ b/acceptance/bundle/override/job_cluster_var/script
@ -0,0 +1,4 @@
+trace $CLI bundle validate -o json -t development | jq '.resources.jobs'
+trace $CLI bundle validate -t development
+trace $CLI bundle validate -o json -t staging | jq '.resources.jobs'
+trace $CLI bundle validate -t staging
--- a/acceptance/bundle/override/job_tasks/databricks.yml
+++ b/acceptance/bundle/override/job_tasks/databricks.yml
@ -1,9 +1,6 @@
 bundle:
  name: override_job_tasks

-workspace:
-  host: https://acme.cloud.databricks.com/
-
 resources:
  jobs:
    foo:
--- a/acceptance/bundle/override/job_tasks/out.development.stderr.txt
+++ b/acceptance/bundle/override/job_tasks/out.development.stderr.txt
@ -0,0 +1,6 @@
+
+>>> errcode $CLI bundle validate -o json -t development
+Error: file ./test1.py not found
+
+
+Exit code: 1
--- a/acceptance/bundle/override/job_tasks/output.txt
+++ b/acceptance/bundle/override/job_tasks/output.txt
@ -0,0 +1,77 @@
+{
+  "name": "job",
+  "queue": {
+    "enabled": true
+  },
+  "tags": {},
+  "tasks": [
+    {
+      "new_cluster": {
+        "node_type_id": "i3.xlarge",
+        "num_workers": 1,
+        "spark_version": "13.3.x-scala2.12"
+      },
+      "spark_python_task": {
+        "python_file": "./test1.py"
+      },
+      "task_key": "key1"
+    },
+    {
+      "new_cluster": {
+        "spark_version": "13.3.x-scala2.12"
+      },
+      "spark_python_task": {
+        "python_file": "./test2.py"
+      },
+      "task_key": "key2"
+    }
+  ]
+}
+
+>>> errcode $CLI bundle validate -o json -t staging
+Error: file ./test1.py not found
+
+
+Exit code: 1
+{
+  "name": "job",
+  "queue": {
+    "enabled": true
+  },
+  "tags": {},
+  "tasks": [
+    {
+      "new_cluster": {
+        "spark_version": "13.3.x-scala2.12"
+      },
+      "spark_python_task": {
+        "python_file": "./test1.py"
+      },
+      "task_key": "key1"
+    },
+    {
+      "new_cluster": {
+        "node_type_id": "i3.2xlarge",
+        "num_workers": 4,
+        "spark_version": "13.3.x-scala2.12"
+      },
+      "spark_python_task": {
+        "python_file": "./test3.py"
+      },
+      "task_key": "key2"
+    }
+  ]
+}
+
+>>> errcode $CLI bundle validate -t staging
+Error: file ./test1.py not found
+
+Name: override_job_tasks
+Target: staging
+Workspace:
+  User: tester@databricks.com
+  Path: /Workspace/Users/tester@databricks.com/.bundle/override_job_tasks/staging
+
+Found 1 error
+
+Exit code: 1
--- a/acceptance/bundle/override/job_tasks/script
+++ b/acceptance/bundle/override/job_tasks/script
@ -0,0 +1,3 @@
+trace errcode $CLI bundle validate -o json -t development 2> out.development.stderr.txt | jq .resources.jobs.foo
+trace errcode $CLI bundle validate -o json -t staging | jq .resources.jobs.foo
+trace errcode $CLI bundle validate -t staging
--- a/acceptance/bundle/override/merge-string-map/databricks.yml
+++ b/acceptance/bundle/override/merge-string-map/databricks.yml
@ -0,0 +1,13 @@
+bundle:
+  name: merge-string-map
+
+resources:
+  clusters:
+    my_cluster: "hello"
+
+targets:
+  dev:
+    resources:
+      clusters:
+        my_cluster:
+          spark_version: "25"
--- a/acceptance/bundle/override/merge-string-map/output.txt
+++ b/acceptance/bundle/override/merge-string-map/output.txt
@ -0,0 +1,27 @@
+
+>>> $CLI bundle validate -o json -t dev
+Warning: expected map, found string
+  at resources.clusters.my_cluster
+  in databricks.yml:6:17
+
+{
+  "clusters": {
+    "my_cluster": {
+      "custom_tags": {},
+      "spark_version": "25"
+    }
+  }
+}
+
+>>> $CLI bundle validate -t dev
+Warning: expected map, found string
+  at resources.clusters.my_cluster
+  in databricks.yml:6:17
+
+Name: merge-string-map
+Target: dev
+Workspace:
+  User: tester@databricks.com
+  Path: /Workspace/Users/tester@databricks.com/.bundle/merge-string-map/dev
+
+Found 1 warning
--- a/acceptance/bundle/override/merge-string-map/script
+++ b/acceptance/bundle/override/merge-string-map/script
@ -0,0 +1,2 @@
+trace $CLI bundle validate -o json -t dev | jq .resources
+trace $CLI bundle validate -t dev
--- a/acceptance/bundle/override/pipeline_cluster/databricks.yml
+++ b/acceptance/bundle/override/pipeline_cluster/databricks.yml
@ -1,9 +1,6 @@
 bundle:
  name: override_pipeline_cluster

-workspace:
-  host: https://acme.cloud.databricks.com/
-
 resources:
  pipelines:
    foo:
--- a/acceptance/bundle/override/pipeline_cluster/output.txt
+++ b/acceptance/bundle/override/pipeline_cluster/output.txt
@ -0,0 +1,44 @@
+
+>>> $CLI bundle validate -o json -t development
+{
+  "foo": {
+    "clusters": [
+      {
+        "label": "default",
+        "node_type_id": "i3.xlarge",
+        "num_workers": 1,
+        "spark_conf": {
+          "foo": "bar"
+        }
+      }
+    ],
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_pipeline_cluster/development/state/metadata.json"
+    },
+    "name": "job",
+    "permissions": []
+  }
+}
+
+>>> $CLI bundle validate -o json -t staging
+{
+  "foo": {
+    "clusters": [
+      {
+        "label": "default",
+        "node_type_id": "i3.2xlarge",
+        "num_workers": 4,
+        "spark_conf": {
+          "foo": "bar"
+        }
+      }
+    ],
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_pipeline_cluster/staging/state/metadata.json"
+    },
+    "name": "job",
+    "permissions": []
+  }
+}
--- a/acceptance/bundle/override/pipeline_cluster/script
+++ b/acceptance/bundle/override/pipeline_cluster/script
@ -0,0 +1,2 @@
+trace $CLI bundle validate -o json -t development | jq .resources.pipelines
+trace $CLI bundle validate -o json -t staging | jq .resources.pipelines
--- a/acceptance/bundle/undefined_resources/databricks.yml
+++ b/acceptance/bundle/undefined_resources/databricks.yml
--- a/acceptance/bundle/undefined_resources/output.txt
+++ b/acceptance/bundle/undefined_resources/output.txt
@ -0,0 +1,19 @@
+Error: experiment undefined-experiment is not defined
+  at resources.experiments.undefined-experiment
+  in databricks.yml:11:26
+
+Error: job undefined-job is not defined
+  at resources.jobs.undefined-job
+  in databricks.yml:6:19
+
+Error: pipeline undefined-pipeline is not defined
+  at resources.pipelines.undefined-pipeline
+  in databricks.yml:14:24
+
+Found 3 errors
+
+Name: undefined-job
+Target: default
+
+
+Exit code: 1
--- a/acceptance/bundle/undefined_resources/script
+++ b/acceptance/bundle/undefined_resources/script
@ -0,0 +1,2 @@
+# We need sort_blocks.py because the order of diagnostics is currently randomized
+$CLI bundle validate 2>&1 | sort_blocks.py
--- a/acceptance/bundle/variables/complex-transitive/databricks.yml
+++ b/acceptance/bundle/variables/complex-transitive/databricks.yml
@ -0,0 +1,19 @@
+bundle:
+  name: complex-transitive
+
+variables:
+  catalog:
+    default: hive_metastore
+  spark_conf:
+    default:
+      "spark.databricks.sql.initial.catalog.name": ${var.catalog}
+  etl_cluster_config:
+    type: complex
+    default:
+      spark_version: 14.3.x-scala2.12
+      runtime_engine: PHOTON
+      spark_conf: ${var.spark_conf}
+
+resources:
+  clusters:
+    my_cluster: ${var.etl_cluster_config}
--- a/acceptance/bundle/variables/complex-transitive/output.txt
+++ b/acceptance/bundle/variables/complex-transitive/output.txt
@ -0,0 +1,3 @@
+{
+  "spark.databricks.sql.initial.catalog.name": "${var.catalog}"
+}
--- a/acceptance/bundle/variables/complex-transitive/script
+++ b/acceptance/bundle/variables/complex-transitive/script
@ -0,0 +1,2 @@
+# Currently, this incorrectly outputs variable reference instead of resolved value
+$CLI bundle validate -o json | jq '.resources.clusters.my_cluster.spark_conf'
--- a/acceptance/bundle/variables/complex/databricks.yml
+++ b/acceptance/bundle/variables/complex/databricks.yml
@ -11,6 +11,7 @@ resources:
      - task_key: test
        job_cluster_key: key
        libraries: ${variables.libraries.value}
+        # specific fields of complex variable are referenced:
        task_key: "task with spark version ${var.cluster.spark_version} and jar ${var.libraries[0].jar}"

 variables:
@ -35,30 +36,21 @@ variables:
      - jar: "/path/to/jar"
      - egg: "/path/to/egg"
      - whl: "/path/to/whl"
-  complexvar:
-    type: complex
-    description: "A complex variable"
-    default:
-      key1: "value1"
-      key2: "value2"
-      key3: "value3"
-

 targets:
  default:
+    default: true
  dev:
    variables:
      node_type: "Standard_DS3_v3"
      cluster:
+        # complex variables are not merged, so missing variables (policy_id) are not inherited
        spark_version: "14.2.x-scala2.11"
        node_type_id: ${var.node_type}
        num_workers: 4
        spark_conf:
          spark.speculation: false
          spark.databricks.delta.retentionDurationCheck.enabled: false
-      complexvar:
-        type: complex
-        default:
-          key1: "1"
-          key2: "2"
-          key3: "3"
+      libraries:
+        - jar: "/newpath/to/jar"
+        - whl: "/newpath/to/whl"
--- a/acceptance/bundle/variables/complex/out.default.json
+++ b/acceptance/bundle/variables/complex/out.default.json
@ -0,0 +1,110 @@
+{
+  "resources": {
+    "jobs": {
+      "my_job": {
+        "deployment": {
+          "kind": "BUNDLE",
+          "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/complex-variables/default/state/metadata.json"
+        },
+        "edit_mode": "UI_LOCKED",
+        "format": "MULTI_TASK",
+        "job_clusters": [
+          {
+            "job_cluster_key": "key",
+            "new_cluster": {
+              "node_type_id": "Standard_DS3_v2",
+              "num_workers": 2,
+              "policy_id": "some-policy-id",
+              "spark_conf": {
+                "spark.databricks.delta.retentionDurationCheck.enabled": "false",
+                "spark.random": "true",
+                "spark.speculation": "true"
+              },
+              "spark_version": "13.2.x-scala2.11"
+            }
+          }
+        ],
+        "permissions": [],
+        "queue": {
+          "enabled": true
+        },
+        "tags": {},
+        "tasks": [
+          {
+            "job_cluster_key": "key",
+            "libraries": [
+              {
+                "jar": "/path/to/jar"
+              },
+              {
+                "egg": "/path/to/egg"
+              },
+              {
+                "whl": "/path/to/whl"
+              }
+            ],
+            "task_key": "task with spark version 13.2.x-scala2.11 and jar /path/to/jar"
+          }
+        ]
+      }
+    }
+  },
+  "variables": {
+    "cluster": {
+      "default": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 2,
+        "policy_id": "some-policy-id",
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.random": true,
+          "spark.speculation": true
+        },
+        "spark_version": "13.2.x-scala2.11"
+      },
+      "description": "A cluster definition",
+      "type": "complex",
+      "value": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 2,
+        "policy_id": "some-policy-id",
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.random": true,
+          "spark.speculation": true
+        },
+        "spark_version": "13.2.x-scala2.11"
+      }
+    },
+    "libraries": {
+      "default": [
+        {
+          "jar": "/path/to/jar"
+        },
+        {
+          "egg": "/path/to/egg"
+        },
+        {
+          "whl": "/path/to/whl"
+        }
+      ],
+      "description": "A libraries definition",
+      "type": "complex",
+      "value": [
+        {
+          "jar": "/path/to/jar"
+        },
+        {
+          "egg": "/path/to/egg"
+        },
+        {
+          "whl": "/path/to/whl"
+        }
+      ]
+    },
+    "node_type": {
+      "default": "Standard_DS3_v2",
+      "value": "Standard_DS3_v2"
+    }
+  }
+}
--- a/acceptance/bundle/variables/complex/out.dev.json
+++ b/acceptance/bundle/variables/complex/out.dev.json
@ -0,0 +1,95 @@
+{
+  "resources": {
+    "jobs": {
+      "my_job": {
+        "deployment": {
+          "kind": "BUNDLE",
+          "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/complex-variables/dev/state/metadata.json"
+        },
+        "edit_mode": "UI_LOCKED",
+        "format": "MULTI_TASK",
+        "job_clusters": [
+          {
+            "job_cluster_key": "key",
+            "new_cluster": {
+              "node_type_id": "Standard_DS3_v3",
+              "num_workers": 4,
+              "spark_conf": {
+                "spark.databricks.delta.retentionDurationCheck.enabled": "false",
+                "spark.speculation": "false"
+              },
+              "spark_version": "14.2.x-scala2.11"
+            }
+          }
+        ],
+        "permissions": [],
+        "queue": {
+          "enabled": true
+        },
+        "tags": {},
+        "tasks": [
+          {
+            "job_cluster_key": "key",
+            "libraries": [
+              {
+                "jar": "/newpath/to/jar"
+              },
+              {
+                "whl": "/newpath/to/whl"
+              }
+            ],
+            "task_key": "task with spark version 14.2.x-scala2.11 and jar /newpath/to/jar"
+          }
+        ]
+      }
+    }
+  },
+  "variables": {
+    "cluster": {
+      "default": {
+        "node_type_id": "Standard_DS3_v3",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      },
+      "description": "A cluster definition",
+      "type": "complex",
+      "value": {
+        "node_type_id": "Standard_DS3_v3",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      }
+    },
+    "libraries": {
+      "default": [
+        {
+          "jar": "/newpath/to/jar"
+        },
+        {
+          "whl": "/newpath/to/whl"
+        }
+      ],
+      "description": "A libraries definition",
+      "type": "complex",
+      "value": [
+        {
+          "jar": "/newpath/to/jar"
+        },
+        {
+          "whl": "/newpath/to/whl"
+        }
+      ]
+    },
+    "node_type": {
+      "default": "Standard_DS3_v3",
+      "value": "Standard_DS3_v3"
+    }
+  }
+}
--- a/acceptance/bundle/variables/complex/output.txt
+++ b/acceptance/bundle/variables/complex/output.txt
@ -0,0 +1,14 @@
+
+>>> $CLI bundle validate -o json
+
+>>> jq .resources.jobs.my_job.tasks[0].task_key out.default.json
+"task with spark version 13.2.x-scala2.11 and jar /path/to/jar"
+
+>>> $CLI bundle validate -o json -t dev
+
+>>> jq .resources.jobs.my_job.tasks[0].task_key out.dev.json
+"task with spark version 14.2.x-scala2.11 and jar /newpath/to/jar"
+policy_id and spark_conf.spark_random fields do not exist in dev target:
+
+>>> jq .resources.jobs.my_job.job_clusters[0].new_cluster.policy_id out.dev.json
+null
--- a/acceptance/bundle/variables/complex/script
+++ b/acceptance/bundle/variables/complex/script
@ -0,0 +1,8 @@
+trace $CLI bundle validate -o json | jq '{resources,variables}' > out.default.json
+trace jq .resources.jobs.my_job.tasks[0].task_key out.default.json | grep "task with spark version 13.2.x-scala2.11 and jar /path/to/jar"
+
+trace $CLI bundle validate -o json -t dev | jq '{resources,variables}' > out.dev.json
+trace jq .resources.jobs.my_job.tasks[0].task_key out.dev.json | grep "task with spark version 14.2.x-scala2.11 and jar /newpath/to/jar"
+
+echo policy_id and spark_conf.spark_random fields do not exist in dev target:
+trace jq .resources.jobs.my_job.job_clusters[0].new_cluster.policy_id out.dev.json | grep null
--- a/acceptance/bundle/variables/complex_multiple_files/databricks.yml
+++ b/acceptance/bundle/variables/complex_multiple_files/databricks.yml
--- a/acceptance/bundle/variables/complex_multiple_files/output.txt
+++ b/acceptance/bundle/variables/complex_multiple_files/output.txt
@ -0,0 +1,159 @@
+{
+  "resources": {
+    "jobs": {
+      "my_job": {
+        "deployment": {
+          "kind": "BUNDLE",
+          "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/complex-variables-multiple-files/dev/state/metadata.json"
+        },
+        "edit_mode": "UI_LOCKED",
+        "format": "MULTI_TASK",
+        "job_clusters": [
+          {
+            "job_cluster_key": "key1",
+            "new_cluster": {
+              "node_type_id": "Standard_DS3_v2",
+              "num_workers": 4,
+              "spark_conf": {
+                "spark.databricks.delta.retentionDurationCheck.enabled": "false",
+                "spark.speculation": "false"
+              },
+              "spark_version": "14.2.x-scala2.11"
+            }
+          },
+          {
+            "job_cluster_key": "key2",
+            "new_cluster": {
+              "node_type_id": "Standard_DS3_v2",
+              "num_workers": 4,
+              "spark_conf": {
+                "spark.databricks.delta.retentionDurationCheck.enabled": "false",
+                "spark.speculation": "false"
+              },
+              "spark_version": "14.2.x-scala2.11"
+            }
+          },
+          {
+            "job_cluster_key": "key3",
+            "new_cluster": {
+              "node_type_id": "Standard_DS3_v2",
+              "num_workers": 4,
+              "spark_conf": {
+                "spark.databricks.delta.retentionDurationCheck.enabled": "false",
+                "spark.speculation": "false"
+              },
+              "spark_version": "14.2.x-scala2.11"
+            }
+          },
+          {
+            "job_cluster_key": "key4",
+            "new_cluster": {
+              "node_type_id": "Standard_DS3_v2",
+              "num_workers": 4,
+              "spark_conf": {
+                "spark.databricks.delta.retentionDurationCheck.enabled": "false",
+                "spark.speculation": "false"
+              },
+              "spark_version": "14.2.x-scala2.11"
+            }
+          }
+        ],
+        "permissions": [],
+        "queue": {
+          "enabled": true
+        },
+        "tags": {}
+      }
+    }
+  },
+  "variables": {
+    "cluster1": {
+      "default": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      },
+      "description": "A cluster definition",
+      "type": "complex",
+      "value": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      }
+    },
+    "cluster2": {
+      "default": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      },
+      "description": "A cluster definition",
+      "type": "complex",
+      "value": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      }
+    },
+    "cluster3": {
+      "default": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      },
+      "description": "A cluster definition",
+      "type": "complex",
+      "value": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      }
+    },
+    "cluster4": {
+      "default": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      },
+      "description": "A cluster definition",
+      "type": "complex",
+      "value": {
+        "node_type_id": "Standard_DS3_v2",
+        "num_workers": 4,
+        "spark_conf": {
+          "spark.databricks.delta.retentionDurationCheck.enabled": false,
+          "spark.speculation": false
+        },
+        "spark_version": "14.2.x-scala2.11"
+      }
+    }
+  }
+}
--- a/acceptance/bundle/variables/complex_multiple_files/script
+++ b/acceptance/bundle/variables/complex_multiple_files/script
@ -0,0 +1 @@
+$CLI bundle validate -t dev -o json | jq '{resources, variables}'
--- a/acceptance/bundle/variables/complex_multiple_files/variables/clusters.yml
+++ b/acceptance/bundle/variables/complex_multiple_files/variables/clusters.yml
--- a/acceptance/bundle/variables/empty/databricks.yml
+++ b/acceptance/bundle/variables/empty/databricks.yml
--- a/acceptance/bundle/variables/empty/output.txt
+++ b/acceptance/bundle/variables/empty/output.txt
@ -0,0 +1,11 @@
+Error: no value assigned to required variable a. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_a environment variable
+
+Name: empty${var.a}
+Target: default
+Workspace:
+  User: tester@databricks.com
+  Path: /Workspace/Users/tester@databricks.com/.bundle/empty${var.a}/default
+
+Found 1 error
+
+Exit code: 1
--- a/acceptance/bundle/variables/empty/script
+++ b/acceptance/bundle/variables/empty/script
@ -0,0 +1 @@
+$CLI bundle validate
--- a/acceptance/bundle/variables/env_overrides/databricks.yml
+++ b/acceptance/bundle/variables/env_overrides/databricks.yml
--- a/acceptance/bundle/variables/env_overrides/output.txt
+++ b/acceptance/bundle/variables/env_overrides/output.txt
@ -0,0 +1,40 @@
+
+>>> $CLI bundle validate -t env-with-single-variable-override -o json
+"default-a dev-b"
+
+>>> $CLI bundle validate -t env-with-two-variable-overrides -o json
+"prod-a prod-b"
+
+>>> BUNDLE_VAR_b=env-var-b $CLI bundle validate -t env-with-two-variable-overrides -o json
+"prod-a env-var-b"
+
+>>> errcode $CLI bundle validate -t env-missing-a-required-variable-assignment
+Error: no value assigned to required variable b. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_b environment variable
+
+Name: test bundle
+Target: env-missing-a-required-variable-assignment
+Workspace:
+  User: tester@databricks.com
+  Path: /Workspace/Users/tester@databricks.com/.bundle/test bundle/env-missing-a-required-variable-assignment
+
+Found 1 error
+
+Exit code: 1
+
+>>> errcode $CLI bundle validate -t env-using-an-undefined-variable
+Error: variable c is not defined but is assigned a value
+
+Name: test bundle
+
+Found 1 error
+
+Exit code: 1
+
+>>> $CLI bundle validate -t env-overrides-lookup -o json
+{
+  "a": "default-a",
+  "b": "prod-b",
+  "d": "4321",
+  "e": "1234",
+  "f": "9876"
+}
--- a/acceptance/bundle/variables/env_overrides/script
+++ b/acceptance/bundle/variables/env_overrides/script
@ -0,0 +1,6 @@
+trace $CLI bundle validate -t env-with-single-variable-override -o json | jq .workspace.profile
+trace $CLI bundle validate -t env-with-two-variable-overrides -o json | jq .workspace.profile
+trace BUNDLE_VAR_b=env-var-b $CLI bundle validate -t env-with-two-variable-overrides -o json | jq .workspace.profile
+trace errcode $CLI bundle validate -t env-missing-a-required-variable-assignment
+trace errcode $CLI bundle validate -t env-using-an-undefined-variable
+trace $CLI bundle validate -t env-overrides-lookup -o json | jq '.variables | map_values(.value)'
--- a/acceptance/bundle/variables/resolve-builtin/databricks.yml
+++ b/acceptance/bundle/variables/resolve-builtin/databricks.yml
@ -0,0 +1,6 @@
+bundle:
+  name: TestResolveVariableReferences
+
+workspace:
+  root_path: "${bundle.name}/bar"
+  file_path: "${workspace.root_path}/baz"
--- a/acceptance/bundle/variables/resolve-builtin/output.txt
+++ b/acceptance/bundle/variables/resolve-builtin/output.txt
@ -0,0 +1,11 @@
+{
+  "artifact_path": "TestResolveVariableReferences/bar/artifacts",
+  "current_user": {
+    "short_name": "tester",
+    "userName": "tester@databricks.com"
+  },
+  "file_path": "TestResolveVariableReferences/bar/baz",
+  "resource_path": "TestResolveVariableReferences/bar/resources",
+  "root_path": "TestResolveVariableReferences/bar",
+  "state_path": "TestResolveVariableReferences/bar/state"
+}
--- a/acceptance/bundle/variables/resolve-builtin/script
+++ b/acceptance/bundle/variables/resolve-builtin/script
@ -0,0 +1 @@
+$CLI bundle validate -o json | jq .workspace
--- a/acceptance/bundle/variables/resolve-empty/databricks.yml
+++ b/acceptance/bundle/variables/resolve-empty/databricks.yml
@ -0,0 +1,10 @@
+bundle:
+  name: TestResolveVariableReferencesToEmptyFields
+  git:
+    branch: ""
+
+resources:
+  jobs:
+    job1:
+      tags:
+        git_branch: "${bundle.git.branch}"
--- a/acceptance/bundle/variables/resolve-empty/output.txt
+++ b/acceptance/bundle/variables/resolve-empty/output.txt
@ -0,0 +1,3 @@
+{
+  "git_branch": ""
+}
--- a/acceptance/bundle/variables/resolve-empty/script
+++ b/acceptance/bundle/variables/resolve-empty/script
@ -0,0 +1 @@
+$CLI bundle validate -o json | jq .resources.jobs.job1.tags
--- a/acceptance/bundle/variables/resolve-field-within-complex/databricks.yml
+++ b/acceptance/bundle/variables/resolve-field-within-complex/databricks.yml
@ -0,0 +1,16 @@
+bundle:
+  name: TestResolveComplexVariableReferencesToFields
+
+variables:
+  cluster:
+    type: "complex"
+    default:
+      node_type_id: "Standard_DS3_v2"
+      num_workers: 2
+
+resources:
+  jobs:
+    job1:
+      job_clusters:
+        - new_cluster:
+            node_type_id: "${var.cluster.node_type_id}"
--- a/acceptance/bundle/variables/resolve-field-within-complex/output.txt
+++ b/acceptance/bundle/variables/resolve-field-within-complex/output.txt
@ -0,0 +1,3 @@
+{
+  "node_type_id": "Standard_DS3_v2"
+}
--- a/acceptance/bundle/variables/resolve-field-within-complex/script
+++ b/acceptance/bundle/variables/resolve-field-within-complex/script
@ -0,0 +1 @@
+$CLI bundle validate -o json | jq .resources.jobs.job1.job_clusters[0].new_cluster
--- a/acceptance/bundle/variables/vanilla/databricks.yml
+++ b/acceptance/bundle/variables/vanilla/databricks.yml
--- a/acceptance/bundle/variables/vanilla/output.txt
+++ b/acceptance/bundle/variables/vanilla/output.txt
@ -0,0 +1,16 @@
+
+>>> BUNDLE_VAR_b=def $CLI bundle validate -o json
+"abc def"
+
+>>> errcode $CLI bundle validate
+Error: no value assigned to required variable b. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_b environment variable
+
+Name: ${var.a} ${var.b}
+Target: default
+Workspace:
+  User: tester@databricks.com
+  Path: /Workspace/Users/tester@databricks.com/.bundle/${var.a} ${var.b}/default
+
+Found 1 error
+
+Exit code: 1
--- a/acceptance/bundle/variables/vanilla/script
+++ b/acceptance/bundle/variables/vanilla/script
@ -0,0 +1,2 @@
+trace BUNDLE_VAR_b=def $CLI bundle validate -o json | jq .bundle.name
+trace errcode $CLI bundle validate
--- a/acceptance/bundle/variables/variable_overrides_in_target/databricks.yml
+++ b/acceptance/bundle/variables/variable_overrides_in_target/databricks.yml
--- a/acceptance/bundle/variables/variable_overrides_in_target/output.txt
+++ b/acceptance/bundle/variables/variable_overrides_in_target/output.txt
@ -0,0 +1,84 @@
+
+>>> $CLI bundle validate -o json -t use-default-variable-values
+{
+  "pipelines": {
+    "my_pipeline": {
+      "clusters": [
+        {
+          "label": "default",
+          "num_workers": 42
+        }
+      ],
+      "continuous": true,
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/foobar/use-default-variable-values/state/metadata.json"
+      },
+      "name": "a_string",
+      "permissions": []
+    }
+  }
+}
+
+>>> $CLI bundle validate -o json -t override-string-variable
+{
+  "pipelines": {
+    "my_pipeline": {
+      "clusters": [
+        {
+          "label": "default",
+          "num_workers": 42
+        }
+      ],
+      "continuous": true,
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/foobar/override-string-variable/state/metadata.json"
+      },
+      "name": "overridden_string",
+      "permissions": []
+    }
+  }
+}
+
+>>> $CLI bundle validate -o json -t override-int-variable
+{
+  "pipelines": {
+    "my_pipeline": {
+      "clusters": [
+        {
+          "label": "default",
+          "num_workers": 43
+        }
+      ],
+      "continuous": true,
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/foobar/override-int-variable/state/metadata.json"
+      },
+      "name": "a_string",
+      "permissions": []
+    }
+  }
+}
+
+>>> $CLI bundle validate -o json -t override-both-bool-and-string-variables
+{
+  "pipelines": {
+    "my_pipeline": {
+      "clusters": [
+        {
+          "label": "default",
+          "num_workers": 42
+        }
+      ],
+      "continuous": false,
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/foobar/override-both-bool-and-string-variables/state/metadata.json"
+      },
+      "name": "overridden_string",
+      "permissions": []
+    }
+  }
+}
--- a/acceptance/bundle/variables/variable_overrides_in_target/script
+++ b/acceptance/bundle/variables/variable_overrides_in_target/script
@ -0,0 +1,4 @@
+trace $CLI bundle validate -o json -t use-default-variable-values | jq .resources
+trace $CLI bundle validate -o json -t override-string-variable | jq .resources
+trace $CLI bundle validate -o json -t override-int-variable | jq .resources
+trace $CLI bundle validate -o json -t override-both-bool-and-string-variables | jq .resources
--- a/acceptance/bundle/variables/without_definition/databricks.yml
+++ b/acceptance/bundle/variables/without_definition/databricks.yml
@ -1,3 +1,5 @@
+bundle:
+  name: x
 variables:
  a:
  b:
--- a/acceptance/bundle/variables/without_definition/output.txt
+++ b/acceptance/bundle/variables/without_definition/output.txt
@ -0,0 +1,4 @@
+{
+  "a": "foo",
+  "b": "bar"
+}
--- a/acceptance/bundle/variables/without_definition/script
+++ b/acceptance/bundle/variables/without_definition/script
@ -0,0 +1 @@
+BUNDLE_VAR_a=foo BUNDLE_VAR_b=bar $CLI bundle validate -o json | jq '.variables | map_values(.value)'
--- a/acceptance/help/output.txt
+++ b/acceptance/help/output.txt
@ -0,0 +1,143 @@
+Databricks CLI
+
+Usage:
+  databricks [command]
+
+Databricks Workspace
+  fs                                     Filesystem related commands
+  git-credentials                        Registers personal access token for Databricks to do operations on behalf of the user.
+  repos                                  The Repos API allows users to manage their git repos.
+  secrets                                The Secrets API allows you to manage secrets, secret scopes, and access permissions.
+  workspace                              The Workspace API allows you to list, import, export, and delete notebooks and folders.
+
+Compute
+  cluster-policies                       You can use cluster policies to control users' ability to configure clusters based on a set of rules.
+  clusters                               The Clusters API allows you to create, start, edit, list, terminate, and delete clusters.
+  global-init-scripts                    The Global Init Scripts API enables Workspace administrators to configure global initialization scripts for their workspace.
+  instance-pools                         Instance Pools API are used to create, edit, delete and list instance pools by using ready-to-use cloud instances which reduces a cluster start and auto-scaling times.
+  instance-profiles                      The Instance Profiles API allows admins to add, list, and remove instance profiles that users can launch clusters with.
+  libraries                              The Libraries API allows you to install and uninstall libraries and get the status of libraries on a cluster.
+  policy-compliance-for-clusters         The policy compliance APIs allow you to view and manage the policy compliance status of clusters in your workspace.
+  policy-families                        View available policy families.
+
+Workflows
+  jobs                                   The Jobs API allows you to create, edit, and delete jobs.
+  policy-compliance-for-jobs             The compliance APIs allow you to view and manage the policy compliance status of jobs in your workspace.
+
+Delta Live Tables
+  pipelines                              The Delta Live Tables API allows you to create, edit, delete, start, and view details about pipelines.
+
+Machine Learning
+  experiments                            Experiments are the primary unit of organization in MLflow; all MLflow runs belong to an experiment.
+  model-registry                         Note: This API reference documents APIs for the Workspace Model Registry.
+
+Real-time Serving
+  serving-endpoints                      The Serving Endpoints API allows you to create, update, and delete model serving endpoints.
+
+Identity and Access Management
+  current-user                           This API allows retrieving information about currently authenticated user or service principal.
+  groups                                 Groups simplify identity management, making it easier to assign access to Databricks workspace, data, and other securable objects.
+  permissions                            Permissions API are used to create read, write, edit, update and manage access for various users on different objects and endpoints.
+  service-principals                     Identities for use with jobs, automated tools, and systems such as scripts, apps, and CI/CD platforms.
+  users                                  User identities recognized by Databricks and represented by email addresses.
+
+Databricks SQL
+  alerts                                 The alerts API can be used to perform CRUD operations on alerts.
+  alerts-legacy                          The alerts API can be used to perform CRUD operations on alerts.
+  dashboards                             In general, there is little need to modify dashboards using the API.
+  data-sources                           This API is provided to assist you in making new query objects.
+  queries                                The queries API can be used to perform CRUD operations on queries.
+  queries-legacy                         These endpoints are used for CRUD operations on query definitions.
+  query-history                          A service responsible for storing and retrieving the list of queries run against SQL endpoints and serverless compute.
+  warehouses                             A SQL warehouse is a compute resource that lets you run SQL commands on data objects within Databricks SQL.
+
+Unity Catalog
+  artifact-allowlists                    In Databricks Runtime 13.3 and above, you can add libraries and init scripts to the allowlist in UC so that users can leverage these artifacts on compute configured with shared access mode.
+  catalogs                               A catalog is the first layer of Unity Catalog’s three-level namespace.
+  connections                            Connections allow for creating a connection to an external data source.
+  credentials                            A credential represents an authentication and authorization mechanism for accessing services on your cloud tenant.
+  external-locations                     An external location is an object that combines a cloud storage path with a storage credential that authorizes access to the cloud storage path.
+  functions                              Functions implement User-Defined Functions (UDFs) in Unity Catalog.
+  grants                                 In Unity Catalog, data is secure by default.
+  metastores                             A metastore is the top-level container of objects in Unity Catalog.
+  model-versions                         Databricks provides a hosted version of MLflow Model Registry in Unity Catalog.
+  online-tables                          Online tables provide lower latency and higher QPS access to data from Delta tables.
+  quality-monitors                       A monitor computes and monitors data or model quality metrics for a table over time.
+  registered-models                      Databricks provides a hosted version of MLflow Model Registry in Unity Catalog.
+  resource-quotas                        Unity Catalog enforces resource quotas on all securable objects, which limits the number of resources that can be created.
+  schemas                                A schema (also called a database) is the second layer of Unity Catalog’s three-level namespace.
+  storage-credentials                    A storage credential represents an authentication and authorization mechanism for accessing data stored on your cloud tenant.
+  system-schemas                         A system schema is a schema that lives within the system catalog.
+  table-constraints                      Primary key and foreign key constraints encode relationships between fields in tables.
+  tables                                 A table resides in the third layer of Unity Catalog’s three-level namespace.
+  temporary-table-credentials            Temporary Table Credentials refer to short-lived, downscoped credentials used to access cloud storage locationswhere table data is stored in Databricks.
+  volumes                                Volumes are a Unity Catalog (UC) capability for accessing, storing, governing, organizing and processing files.
+  workspace-bindings                     A securable in Databricks can be configured as __OPEN__ or __ISOLATED__.
+
+Delta Sharing
+  providers                              A data provider is an object representing the organization in the real world who shares the data.
+  recipient-activation                   The Recipient Activation API is only applicable in the open sharing model where the recipient object has the authentication type of TOKEN.
+  recipients                             A recipient is an object you create using :method:recipients/create to represent an organization which you want to allow access shares.
+  shares                                 A share is a container instantiated with :method:shares/create.
+
+Settings
+  ip-access-lists                        IP Access List enables admins to configure IP access lists.
+  notification-destinations              The notification destinations API lets you programmatically manage a workspace's notification destinations.
+  settings                               Workspace Settings API allows users to manage settings at the workspace level.
+  token-management                       Enables administrators to get all tokens and delete tokens for other users.
+  tokens                                 The Token API allows you to create, list, and revoke tokens that can be used to authenticate and access Databricks REST APIs.
+  workspace-conf                         This API allows updating known workspace settings for advanced users.
+
+Developer Tools
+  bundle                                 Databricks Asset Bundles let you express data/AI/analytics projects as code.
+  sync                                   Synchronize a local directory to a workspace directory
+
+Vector Search
+  vector-search-endpoints                **Endpoint**: Represents the compute resources to host vector search indexes.
+  vector-search-indexes                  **Index**: An efficient representation of your embedding vectors that supports real-time and efficient approximate nearest neighbor (ANN) search queries.
+
+Dashboards
+  lakeview                               These APIs provide specific management operations for Lakeview dashboards.
+
+Marketplace
+  consumer-fulfillments                  Fulfillments are entities that allow consumers to preview installations.
+  consumer-installations                 Installations are entities that allow consumers to interact with Databricks Marketplace listings.
+  consumer-listings                      Listings are the core entities in the Marketplace.
+  consumer-personalization-requests      Personalization Requests allow customers to interact with the individualized Marketplace listing flow.
+  consumer-providers                     Providers are the entities that publish listings to the Marketplace.
+  provider-exchange-filters              Marketplace exchanges filters curate which groups can access an exchange.
+  provider-exchanges                     Marketplace exchanges allow providers to share their listings with a curated set of customers.
+  provider-files                         Marketplace offers a set of file APIs for various purposes such as preview notebooks and provider icons.
+  provider-listings                      Listings are the core entities in the Marketplace.
+  provider-personalization-requests      Personalization requests are an alternate to instantly available listings.
+  provider-provider-analytics-dashboards Manage templated analytics solution for providers.
+  provider-providers                     Providers are entities that manage assets in Marketplace.
+
+Apps
+  apps                                   Apps run directly on a customer’s Databricks instance, integrate with their data, use and extend Databricks services, and enable users to interact through single sign-on.
+  apps                                   Apps run directly on a customer’s Databricks instance, integrate with their data, use and extend Databricks services, and enable users to interact through single sign-on.
+
+Clean Rooms
+  clean-room-assets                      Clean room assets are data and code objects — Tables, volumes, and notebooks that are shared with the clean room.
+  clean-room-task-runs                   Clean room task runs are the executions of notebooks in a clean room.
+  clean-rooms                            A clean room uses Delta Sharing and serverless compute to provide a secure and privacy-protecting environment where multiple parties can work together on sensitive enterprise data without direct access to each other’s data.
+
+Additional Commands:
+  account                                Databricks Account Commands
+  api                                    Perform Databricks API call
+  auth                                   Authentication related commands
+  completion                             Generate the autocompletion script for the specified shell
+  configure                              Configure authentication
+  help                                   Help about any command
+  labs                                   Manage Databricks Labs installations
+  version                                Retrieve information about the current version of this CLI
+
+Flags:
+      --debug            enable debug logging
+  -h, --help             help for databricks
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+  -v, --version          version for databricks
+
+Use "databricks [command] --help" for more information about a command.
--- a/acceptance/help/script
+++ b/acceptance/help/script
@ -0,0 +1 @@
+$CLI
--- a/acceptance/script.cleanup
+++ b/acceptance/script.cleanup
@ -0,0 +1 @@
+rm -fr .databricks .gitignore
--- a/acceptance/script.prepare
+++ b/acceptance/script.prepare
@ -0,0 +1,36 @@
+# Prevent CLI from downloading terraform in each test:
+export DATABRICKS_TF_EXEC_PATH=/tmp/
+
+errcode() {
+    # Temporarily disable 'set -e' to prevent the script from exiting on error
+    set +e
+    # Execute the provided command with all arguments
+    "$@"
+    local exit_code=$?
+    # Re-enable 'set -e' if it was previously set
+    set -e
+    >&2 printf "\nExit code: $exit_code\n"
+}
+
+trace() {
+    >&2 printf "\n>>> %s\n" "$*"
+
+    if [[ "$1" == *"="* ]]; then
+        # If the first argument contains '=', collect all env vars
+        local env_vars=()
+        while [[ "$1" == *"="* ]]; do
+            env_vars+=("$1")
+            shift
+        done
+        # Export environment variables in a subshell and execute the command
+        (
+            export "${env_vars[@]}"
+            "$@"
+        )
+    else
+        # Execute the command normally
+        "$@"
+    fi
+
+    return $?
+}
--- a/acceptance/server_test.go
+++ b/acceptance/server_test.go
@ -0,0 +1,129 @@
+package acceptance_test
+
+import (
+	"encoding/json"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/service/compute"
+	"github.com/databricks/databricks-sdk-go/service/iam"
+	"github.com/databricks/databricks-sdk-go/service/workspace"
+)
+
+type TestServer struct {
+	*httptest.Server
+	Mux  *http.ServeMux
+	Port int
+}
+
+type HandlerFunc func(r *http.Request) (any, error)
+
+func NewTestServer() *TestServer {
+	mux := http.NewServeMux()
+	server := httptest.NewServer(mux)
+	port := server.Listener.Addr().(*net.TCPAddr).Port
+
+	return &TestServer{
+		Server: server,
+		Mux:    mux,
+		Port:   port,
+	}
+}
+
+func (s *TestServer) Handle(pattern string, handler HandlerFunc) {
+	s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) {
+		resp, err := handler(r)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+
+		var respBytes []byte
+
+		respString, ok := resp.(string)
+		if ok {
+			respBytes = []byte(respString)
+		} else {
+			respBytes, err = json.MarshalIndent(resp, "", "    ")
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
+		}
+
+		if _, err := w.Write(respBytes); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	})
+}
+
+func StartServer(t *testing.T) *TestServer {
+	server := NewTestServer()
+	t.Cleanup(func() {
+		server.Close()
+	})
+	return server
+}
+
+func AddHandlers(server *TestServer) {
+	server.Handle("/api/2.0/policies/clusters/list", func(r *http.Request) (any, error) {
+		return compute.ListPoliciesResponse{
+			Policies: []compute.Policy{
+				{
+					PolicyId: "5678",
+					Name:     "wrong-cluster-policy",
+				},
+				{
+					PolicyId: "9876",
+					Name:     "some-test-cluster-policy",
+				},
+			},
+		}, nil
+	})
+
+	server.Handle("/api/2.0/instance-pools/list", func(r *http.Request) (any, error) {
+		return compute.ListInstancePools{
+			InstancePools: []compute.InstancePoolAndStats{
+				{
+					InstancePoolName: "some-test-instance-pool",
+					InstancePoolId:   "1234",
+				},
+			},
+		}, nil
+	})
+
+	server.Handle("/api/2.1/clusters/list", func(r *http.Request) (any, error) {
+		return compute.ListClustersResponse{
+			Clusters: []compute.ClusterDetails{
+				{
+					ClusterName: "some-test-cluster",
+					ClusterId:   "4321",
+				},
+				{
+					ClusterName: "some-other-cluster",
+					ClusterId:   "9876",
+				},
+			},
+		}, nil
+	})
+
+	server.Handle("/api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, error) {
+		return iam.User{
+			UserName: "tester@databricks.com",
+		}, nil
+	})
+
+	server.Handle("/api/2.0/workspace/get-status", func(r *http.Request) (any, error) {
+		return workspace.ObjectInfo{
+			ObjectId:   1001,
+			ObjectType: "DIRECTORY",
+			Path:       "",
+			ResourceId: "1001",
+		}, nil
+	})
+}
--- a/bundle/artifacts/expand_globs.go
+++ b/bundle/artifacts/expand_globs.go
@ -97,7 +97,7 @@ func (m *expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost
 		return dyn.SetByPath(v, base, dyn.V(output))
 	})
 	if err != nil {
-		return diag.FromErr(err)
+		diags = diags.Extend(diag.FromErr(err))
 	}

 	return diags
--- a/bundle/artifacts/expand_globs_test.go
+++ b/bundle/artifacts/expand_globs_test.go
@ -2,7 +2,6 @@ package artifacts

 import (
 	"context"
-	"fmt"
 	"path/filepath"
 	"testing"

@ -88,16 +87,16 @@ func TestExpandGlobs_InvalidPattern(t *testing.T) {
 	))

 	assert.Len(t, diags, 4)
-	assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("a[.txt")), diags[0].Summary)
+	assert.Equal(t, filepath.Clean("a[.txt")+": syntax error in pattern", diags[0].Summary)
 	assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[0].Locations[0].File)
 	assert.Equal(t, "artifacts.test.files[0].source", diags[0].Paths[0].String())
-	assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("a[.txt")), diags[1].Summary)
+	assert.Equal(t, filepath.Clean("a[.txt")+": syntax error in pattern", diags[1].Summary)
 	assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[1].Locations[0].File)
 	assert.Equal(t, "artifacts.test.files[1].source", diags[1].Paths[0].String())
-	assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("../a[.txt")), diags[2].Summary)
+	assert.Equal(t, filepath.Clean("../a[.txt")+": syntax error in pattern", diags[2].Summary)
 	assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[2].Locations[0].File)
 	assert.Equal(t, "artifacts.test.files[2].source", diags[2].Paths[0].String())
-	assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("subdir/a[.txt")), diags[3].Summary)
+	assert.Equal(t, filepath.Clean("subdir/a[.txt")+": syntax error in pattern", diags[3].Summary)
 	assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[3].Locations[0].File)
 	assert.Equal(t, "artifacts.test.files[3].source", diags[3].Paths[0].String())
 }
--- a/bundle/artifacts/whl/infer.go
+++ b/bundle/artifacts/whl/infer.go
@ -32,7 +32,7 @@ func (m *infer) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	//)

 	py := python.GetExecutable()
-	artifact.BuildCommand = fmt.Sprintf(`%s setup.py bdist_wheel`, py)
+	artifact.BuildCommand = py + " setup.py bdist_wheel"

 	return nil
 }
--- a/bundle/bundle.go
+++ b/bundle/bundle.go
@ -8,6 +8,7 @@ package bundle

 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@ -234,7 +235,7 @@ func (b *Bundle) GetSyncIncludePatterns(ctx context.Context) ([]string, error) {
 // we call into from this bundle context.
 func (b *Bundle) AuthEnv() (map[string]string, error) {
 	if b.client == nil {
-		return nil, fmt.Errorf("workspace client not initialized yet")
+		return nil, errors.New("workspace client not initialized yet")
 	}

 	cfg := b.client.Config
--- a/bundle/bundle_test.go
+++ b/bundle/bundle_test.go
@ -2,7 +2,6 @@ package bundle

 import (
 	"context"
-	"errors"
 	"io/fs"
 	"os"
 	"path/filepath"
@ -16,7 +15,7 @@ import (

 func TestLoadNotExists(t *testing.T) {
 	b, err := Load(context.Background(), "/doesntexist")
-	assert.True(t, errors.Is(err, fs.ErrNotExist))
+	assert.ErrorIs(t, err, fs.ErrNotExist)
 	assert.Nil(t, b)
 }

--- a/bundle/config/artifact.go
+++ b/bundle/config/artifact.go
@ -2,7 +2,7 @@ package config

 import (
 	"context"
-	"fmt"
+	"errors"

 	"github.com/databricks/cli/libs/exec"
 )
@ -37,7 +37,7 @@ type Artifact struct {

 func (a *Artifact) Build(ctx context.Context) ([]byte, error) {
 	if a.BuildCommand == "" {
-		return nil, fmt.Errorf("no build property defined")
+		return nil, errors.New("no build property defined")
 	}

 	var e *exec.Executor
--- a/bundle/config/experimental.go
+++ b/bundle/config/experimental.go
@ -27,9 +27,33 @@ type Experimental struct {
 	// PyDABs determines whether to load the 'databricks-pydabs' package.
 	//
 	// PyDABs allows to define bundle configuration using Python.
+	// PyDABs is deprecated use Python instead.
 	PyDABs PyDABs `json:"pydabs,omitempty"`
+
+	// Python configures loading of Python code defined with 'databricks-bundles' package.
+	Python Python `json:"python,omitempty"`
 }

+type Python struct {
+	// Resources contains a list of fully qualified function paths to load resources
+	// defined in Python code.
+	//
+	// Example: ["my_project.resources:load_resources"]
+	Resources []string `json:"resources"`
+
+	// Mutators contains a list of fully qualified function paths to mutator functions.
+	//
+	// Example: ["my_project.mutators:add_default_cluster"]
+	Mutators []string `json:"mutators"`
+
+	// VEnvPath is path to the virtual environment.
+	//
+	// If enabled, Python code will execute within this environment. If disabled,
+	// it defaults to using the Python interpreter available in the current shell.
+	VEnvPath string `json:"venv_path,omitempty"`
+}
+
+// PyDABs is deprecated use Python instead
 type PyDABs struct {
 	// Enabled is a flag to enable the feature.
 	Enabled bool `json:"enabled,omitempty"`
--- a/bundle/config/mutator/apply_presets.go
+++ b/bundle/config/mutator/apply_presets.go
@ -9,7 +9,6 @@ import (

 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
-	"github.com/databricks/cli/libs/dbr"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/textutil"
@ -222,27 +221,6 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 		dashboard.DisplayName = prefix + dashboard.DisplayName
 	}

-	if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) {
-		isDatabricksWorkspace := dbr.RunsOnRuntime(ctx) && strings.HasPrefix(b.SyncRootPath, "/Workspace/")
-		if !isDatabricksWorkspace {
-			target := b.Config.Bundle.Target
-			path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment"))
-			diags = diags.Append(
-				diag.Diagnostic{
-					Severity: diag.Warning,
-					Summary:  "source-linked deployment is available only in the Databricks Workspace",
-					Paths: []dyn.Path{
-						path,
-					},
-					Locations: b.Config.GetLocations(path[2:].String()),
-				},
-			)
-
-			disabled := false
-			b.Config.Presets.SourceLinkedDeployment = &disabled
-		}
-	}
-
 	return diags
 }

--- a/bundle/config/mutator/apply_presets_test.go
+++ b/bundle/config/mutator/apply_presets_test.go
@ -2,16 +2,12 @@ package mutator_test

 import (
 	"context"
-	"runtime"
 	"testing"

 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/mutator"
 	"github.com/databricks/cli/bundle/config/resources"
-	"github.com/databricks/cli/bundle/internal/bundletest"
-	"github.com/databricks/cli/libs/dbr"
-	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/databricks-sdk-go/service/catalog"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/stretchr/testify/require"
@ -398,87 +394,3 @@ func TestApplyPresetsResourceNotDefined(t *testing.T) {
 		})
 	}
 }
-
-func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
-	}
-
-	testContext := context.Background()
-	enabled := true
-	disabled := false
-	workspacePath := "/Workspace/user.name@company.com"
-
-	tests := []struct {
-		bundlePath      string
-		ctx             context.Context
-		name            string
-		initialValue    *bool
-		expectedValue   *bool
-		expectedWarning string
-	}{
-		{
-			name:          "preset enabled, bundle in Workspace, databricks runtime",
-			bundlePath:    workspacePath,
-			ctx:           dbr.MockRuntime(testContext, true),
-			initialValue:  &enabled,
-			expectedValue: &enabled,
-		},
-		{
-			name:            "preset enabled, bundle not in Workspace, databricks runtime",
-			bundlePath:      "/Users/user.name@company.com",
-			ctx:             dbr.MockRuntime(testContext, true),
-			initialValue:    &enabled,
-			expectedValue:   &disabled,
-			expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
-		},
-		{
-			name:            "preset enabled, bundle in Workspace, not databricks runtime",
-			bundlePath:      workspacePath,
-			ctx:             dbr.MockRuntime(testContext, false),
-			initialValue:    &enabled,
-			expectedValue:   &disabled,
-			expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
-		},
-		{
-			name:          "preset disabled, bundle in Workspace, databricks runtime",
-			bundlePath:    workspacePath,
-			ctx:           dbr.MockRuntime(testContext, true),
-			initialValue:  &disabled,
-			expectedValue: &disabled,
-		},
-		{
-			name:          "preset nil, bundle in Workspace, databricks runtime",
-			bundlePath:    workspacePath,
-			ctx:           dbr.MockRuntime(testContext, true),
-			initialValue:  nil,
-			expectedValue: nil,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			b := &bundle.Bundle{
-				SyncRootPath: tt.bundlePath,
-				Config: config.Root{
-					Presets: config.Presets{
-						SourceLinkedDeployment: tt.initialValue,
-					},
-				},
-			}
-
-			bundletest.SetLocation(b, "presets.source_linked_deployment", []dyn.Location{{File: "databricks.yml"}})
-			diags := bundle.Apply(tt.ctx, b, mutator.ApplyPresets())
-			if diags.HasError() {
-				t.Fatalf("unexpected error: %v", diags)
-			}
-
-			if tt.expectedWarning != "" {
-				require.Equal(t, tt.expectedWarning, diags[0].Summary)
-				require.NotEmpty(t, diags[0].Locations)
-			}
-
-			require.Equal(t, tt.expectedValue, b.Config.Presets.SourceLinkedDeployment)
-		})
-	}
-}
--- a/bundle/config/mutator/apply_source_linked_deployment_preset.go
+++ b/bundle/config/mutator/apply_source_linked_deployment_preset.go
@ -0,0 +1,75 @@
+package mutator
+
+import (
+	"context"
+	"strings"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/libs/dbr"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+)
+
+type applySourceLinkedDeploymentPreset struct{}
+
+// Apply source-linked deployment preset
+func ApplySourceLinkedDeploymentPreset() *applySourceLinkedDeploymentPreset {
+	return &applySourceLinkedDeploymentPreset{}
+}
+
+func (m *applySourceLinkedDeploymentPreset) Name() string {
+	return "ApplySourceLinkedDeploymentPreset"
+}
+
+func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	if config.IsExplicitlyDisabled(b.Config.Presets.SourceLinkedDeployment) {
+		return nil
+	}
+
+	var diags diag.Diagnostics
+	isDatabricksWorkspace := dbr.RunsOnRuntime(ctx) && strings.HasPrefix(b.SyncRootPath, "/Workspace/")
+	target := b.Config.Bundle.Target
+
+	if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) {
+		if !isDatabricksWorkspace {
+			path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment"))
+			diags = diags.Append(
+				diag.Diagnostic{
+					Severity: diag.Warning,
+					Summary:  "source-linked deployment is available only in the Databricks Workspace",
+					Paths: []dyn.Path{
+						path,
+					},
+					Locations: b.Config.GetLocations(path[2:].String()),
+				},
+			)
+
+			disabled := false
+			b.Config.Presets.SourceLinkedDeployment = &disabled
+			return diags
+		}
+	}
+
+	if isDatabricksWorkspace && b.Config.Bundle.Mode == config.Development {
+		enabled := true
+		b.Config.Presets.SourceLinkedDeployment = &enabled
+	}
+
+	if b.Config.Workspace.FilePath != "" && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
+		path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("workspace"), dyn.Key("file_path"))
+
+		diags = diags.Append(
+			diag.Diagnostic{
+				Severity: diag.Warning,
+				Summary:  "workspace.file_path setting will be ignored in source-linked deployment mode",
+				Paths: []dyn.Path{
+					path[2:],
+				},
+				Locations: b.Config.GetLocations(path[2:].String()),
+			},
+		)
+	}
+
+	return diags
+}
--- a/bundle/config/mutator/apply_source_linked_deployment_preset_test.go
+++ b/bundle/config/mutator/apply_source_linked_deployment_preset_test.go
@ -0,0 +1,122 @@
+package mutator_test
+
+import (
+	"context"
+	"runtime"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/cli/bundle/internal/bundletest"
+	"github.com/databricks/cli/libs/dbr"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/stretchr/testify/require"
+)
+
+func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
+	}
+
+	testContext := context.Background()
+	enabled := true
+	disabled := false
+	workspacePath := "/Workspace/user.name@company.com"
+
+	tests := []struct {
+		name            string
+		ctx             context.Context
+		mutateBundle    func(b *bundle.Bundle)
+		initialValue    *bool
+		expectedValue   *bool
+		expectedWarning string
+	}{
+		{
+			name:          "preset enabled, bundle in Workspace, databricks runtime",
+			ctx:           dbr.MockRuntime(testContext, true),
+			initialValue:  &enabled,
+			expectedValue: &enabled,
+		},
+		{
+			name: "preset enabled, bundle not in Workspace, databricks runtime",
+			ctx:  dbr.MockRuntime(testContext, true),
+			mutateBundle: func(b *bundle.Bundle) {
+				b.SyncRootPath = "/Users/user.name@company.com"
+			},
+			initialValue:    &enabled,
+			expectedValue:   &disabled,
+			expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
+		},
+		{
+			name:            "preset enabled, bundle in Workspace, not databricks runtime",
+			ctx:             dbr.MockRuntime(testContext, false),
+			initialValue:    &enabled,
+			expectedValue:   &disabled,
+			expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
+		},
+		{
+			name:          "preset disabled, bundle in Workspace, databricks runtime",
+			ctx:           dbr.MockRuntime(testContext, true),
+			initialValue:  &disabled,
+			expectedValue: &disabled,
+		},
+		{
+			name:          "preset nil, bundle in Workspace, databricks runtime",
+			ctx:           dbr.MockRuntime(testContext, true),
+			initialValue:  nil,
+			expectedValue: nil,
+		},
+		{
+			name: "preset nil, dev mode true, bundle in Workspace, databricks runtime",
+			ctx:  dbr.MockRuntime(testContext, true),
+			mutateBundle: func(b *bundle.Bundle) {
+				b.Config.Bundle.Mode = config.Development
+			},
+			initialValue:  nil,
+			expectedValue: &enabled,
+		},
+		{
+			name: "preset enabled, workspace.file_path is defined by user",
+			ctx:  dbr.MockRuntime(testContext, true),
+			mutateBundle: func(b *bundle.Bundle) {
+				b.Config.Workspace.FilePath = "file_path"
+			},
+			initialValue:    &enabled,
+			expectedValue:   &enabled,
+			expectedWarning: "workspace.file_path setting will be ignored in source-linked deployment mode",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			b := &bundle.Bundle{
+				SyncRootPath: workspacePath,
+				Config: config.Root{
+					Presets: config.Presets{
+						SourceLinkedDeployment: tt.initialValue,
+					},
+				},
+			}
+
+			if tt.mutateBundle != nil {
+				tt.mutateBundle(b)
+			}
+
+			bundletest.SetLocation(b, "presets.source_linked_deployment", []dyn.Location{{File: "databricks.yml"}})
+			bundletest.SetLocation(b, "workspace.file_path", []dyn.Location{{File: "databricks.yml"}})
+
+			diags := bundle.Apply(tt.ctx, b, mutator.ApplySourceLinkedDeploymentPreset())
+			if diags.HasError() {
+				t.Fatalf("unexpected error: %v", diags)
+			}
+
+			if tt.expectedWarning != "" {
+				require.Equal(t, tt.expectedWarning, diags[0].Summary)
+				require.NotEmpty(t, diags[0].Locations)
+			}
+
+			require.Equal(t, tt.expectedValue, b.Config.Presets.SourceLinkedDeployment)
+		})
+	}
+}
--- a/bundle/config/mutator/configure_dashboard_defaults_test.go
+++ b/bundle/config/mutator/configure_dashboard_defaults_test.go
@ -109,19 +109,19 @@ func TestConfigureDashboardDefaultsEmbedCredentials(t *testing.T) {
 	// Set to true; still true.
 	v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d1.embed_credentials")
 	if assert.NoError(t, err) {
-		assert.Equal(t, true, v.MustBool())
+		assert.True(t, v.MustBool())
 	}

 	// Set to false; still false.
 	v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d2.embed_credentials")
 	if assert.NoError(t, err) {
-		assert.Equal(t, false, v.MustBool())
+		assert.False(t, v.MustBool())
 	}

 	// Not set; now false.
 	v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d3.embed_credentials")
 	if assert.NoError(t, err) {
-		assert.Equal(t, false, v.MustBool())
+		assert.False(t, v.MustBool())
 	}

 	// No valid dashboard; no change.
--- a/bundle/config/mutator/default_queueing_test.go
+++ b/bundle/config/mutator/default_queueing_test.go
@ -28,8 +28,8 @@ func TestDefaultQueueingApplyNoJobs(t *testing.T) {
 		},
 	}
 	d := bundle.Apply(context.Background(), b, DefaultQueueing())
-	assert.Len(t, d, 0)
-	assert.Len(t, b.Config.Resources.Jobs, 0)
+	assert.Empty(t, d)
+	assert.Empty(t, b.Config.Resources.Jobs)
 }

 func TestDefaultQueueingApplyJobsAlreadyEnabled(t *testing.T) {
@ -47,7 +47,7 @@ func TestDefaultQueueingApplyJobsAlreadyEnabled(t *testing.T) {
 		},
 	}
 	d := bundle.Apply(context.Background(), b, DefaultQueueing())
-	assert.Len(t, d, 0)
+	assert.Empty(t, d)
 	assert.True(t, b.Config.Resources.Jobs["job"].Queue.Enabled)
 }

@ -66,7 +66,7 @@ func TestDefaultQueueingApplyEnableQueueing(t *testing.T) {
 		},
 	}
 	d := bundle.Apply(context.Background(), b, DefaultQueueing())
-	assert.Len(t, d, 0)
+	assert.Empty(t, d)
 	assert.NotNil(t, b.Config.Resources.Jobs["job"].Queue)
 	assert.True(t, b.Config.Resources.Jobs["job"].Queue.Enabled)
 }
@ -96,7 +96,7 @@ func TestDefaultQueueingApplyWithMultipleJobs(t *testing.T) {
 		},
 	}
 	d := bundle.Apply(context.Background(), b, DefaultQueueing())
-	assert.Len(t, d, 0)
+	assert.Empty(t, d)
 	assert.False(t, b.Config.Resources.Jobs["job1"].Queue.Enabled)
 	assert.True(t, b.Config.Resources.Jobs["job2"].Queue.Enabled)
 	assert.True(t, b.Config.Resources.Jobs["job3"].Queue.Enabled)
--- a/bundle/config/mutator/environments_compat_test.go
+++ b/bundle/config/mutator/environments_compat_test.go
@ -44,7 +44,7 @@ func TestEnvironmentsToTargetsWithEnvironmentsDefined(t *testing.T) {

 	diags := bundle.Apply(context.Background(), b, mutator.EnvironmentsToTargets())
 	require.NoError(t, diags.Error())
-	assert.Len(t, b.Config.Environments, 0)
+	assert.Empty(t, b.Config.Environments)
 	assert.Len(t, b.Config.Targets, 1)
 }

@ -61,6 +61,6 @@ func TestEnvironmentsToTargetsWithTargetsDefined(t *testing.T) {

 	diags := bundle.Apply(context.Background(), b, mutator.EnvironmentsToTargets())
 	require.NoError(t, diags.Error())
-	assert.Len(t, b.Config.Environments, 0)
+	assert.Empty(t, b.Config.Environments)
 	assert.Len(t, b.Config.Targets, 1)
 }
--- a/bundle/config/mutator/expand_workspace_root.go
+++ b/bundle/config/mutator/expand_workspace_root.go
@ -2,7 +2,6 @@ package mutator

 import (
 	"context"
-	"fmt"
 	"path"
 	"strings"

@ -33,7 +32,7 @@ func (m *expandWorkspaceRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.
 	}

 	if strings.HasPrefix(root, "~/") {
-		home := fmt.Sprintf("/Workspace/Users/%s", currentUser.UserName)
+		home := "/Workspace/Users/" + currentUser.UserName
 		b.Config.Workspace.RootPath = path.Join(home, root[2:])
 	}

--- a/bundle/config/mutator/merge_job_tasks_test.go
+++ b/bundle/config/mutator/merge_job_tasks_test.go
@ -74,8 +74,8 @@ func TestMergeJobTasks(t *testing.T) {
 	assert.Equal(t, "i3.2xlarge", cluster.NodeTypeId)
 	assert.Equal(t, 4, cluster.NumWorkers)
 	assert.Len(t, task0.Libraries, 2)
-	assert.Equal(t, task0.Libraries[0].Whl, "package1")
-	assert.Equal(t, task0.Libraries[1].Pypi.Package, "package2")
+	assert.Equal(t, "package1", task0.Libraries[0].Whl)
+	assert.Equal(t, "package2", task0.Libraries[1].Pypi.Package)

 	// This task was left untouched.
 	task1 := j.Tasks[1].NewCluster
--- a/bundle/config/mutator/prepend_workspace_prefix.go
+++ b/bundle/config/mutator/prepend_workspace_prefix.go
@ -55,7 +55,7 @@ func (m *prependWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) di
 					}
 				}

-				return dyn.NewValue(fmt.Sprintf("/Workspace%s", path), v.Locations()), nil
+				return dyn.NewValue("/Workspace"+path, v.Locations()), nil
 			})
 			if err != nil {
 				return dyn.InvalidValue, err
--- a/bundle/config/mutator/process_target_mode.go
+++ b/bundle/config/mutator/process_target_mode.go
@ -6,7 +6,6 @@ import (

 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
-	"github.com/databricks/cli/libs/dbr"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/iamutil"
@ -58,14 +57,6 @@ func transformDevelopmentMode(ctx context.Context, b *bundle.Bundle) {
 		t.TriggerPauseStatus = config.Paused
 	}

-	if !config.IsExplicitlyDisabled(t.SourceLinkedDeployment) {
-		isInWorkspace := strings.HasPrefix(b.SyncRootPath, "/Workspace/")
-		if isInWorkspace && dbr.RunsOnRuntime(ctx) {
-			enabled := true
-			t.SourceLinkedDeployment = &enabled
-		}
-	}
-
 	if !config.IsExplicitlyDisabled(t.PipelinesDevelopment) {
 		enabled := true
 		t.PipelinesDevelopment = &enabled
--- a/bundle/config/mutator/process_target_mode_test.go
+++ b/bundle/config/mutator/process_target_mode_test.go
@ -3,14 +3,12 @@ package mutator
 import (
 	"context"
 	"reflect"
-	"runtime"
 	"slices"
 	"testing"

 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/resources"
-	"github.com/databricks/cli/libs/dbr"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/tags"
 	"github.com/databricks/cli/libs/vfs"
@ -163,18 +161,18 @@ func TestProcessTargetModeDevelopment(t *testing.T) {

 	// Job 1
 	assert.Equal(t, "[dev lennart] job1", b.Config.Resources.Jobs["job1"].Name)
-	assert.Equal(t, b.Config.Resources.Jobs["job1"].Tags["existing"], "tag")
-	assert.Equal(t, b.Config.Resources.Jobs["job1"].Tags["dev"], "lennart")
-	assert.Equal(t, b.Config.Resources.Jobs["job1"].Schedule.PauseStatus, jobs.PauseStatusPaused)
+	assert.Equal(t, "tag", b.Config.Resources.Jobs["job1"].Tags["existing"])
+	assert.Equal(t, "lennart", b.Config.Resources.Jobs["job1"].Tags["dev"])
+	assert.Equal(t, jobs.PauseStatusPaused, b.Config.Resources.Jobs["job1"].Schedule.PauseStatus)

 	// Job 2
 	assert.Equal(t, "[dev lennart] job2", b.Config.Resources.Jobs["job2"].Name)
-	assert.Equal(t, b.Config.Resources.Jobs["job2"].Tags["dev"], "lennart")
-	assert.Equal(t, b.Config.Resources.Jobs["job2"].Schedule.PauseStatus, jobs.PauseStatusUnpaused)
+	assert.Equal(t, "lennart", b.Config.Resources.Jobs["job2"].Tags["dev"])
+	assert.Equal(t, jobs.PauseStatusUnpaused, b.Config.Resources.Jobs["job2"].Schedule.PauseStatus)

 	// Pipeline 1
 	assert.Equal(t, "[dev lennart] pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name)
-	assert.Equal(t, false, b.Config.Resources.Pipelines["pipeline1"].Continuous)
+	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].Continuous)
 	assert.True(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)

 	// Experiment 1
@ -382,7 +380,7 @@ func TestAllResourcesMocked(t *testing.T) {
 	b := mockBundle(config.Development)
 	resources := reflect.ValueOf(b.Config.Resources)

-	for i := 0; i < resources.NumField(); i++ {
+	for i := range resources.NumField() {
 		field := resources.Field(i)
 		if field.Kind() == reflect.Map {
 			assert.True(
@ -411,7 +409,7 @@ func TestAllNonUcResourcesAreRenamed(t *testing.T) {
 	require.NoError(t, diags.Error())

 	resources := reflect.ValueOf(b.Config.Resources)
-	for i := 0; i < resources.NumField(); i++ {
+	for i := range resources.NumField() {
 		field := resources.Field(i)

 		if field.Kind() == reflect.Map {
@ -540,32 +538,3 @@ func TestPipelinesDevelopmentDisabled(t *testing.T) {

 	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
 }
-
-func TestSourceLinkedDeploymentEnabled(t *testing.T) {
-	b, diags := processSourceLinkedBundle(t, true)
-	require.NoError(t, diags.Error())
-	assert.True(t, *b.Config.Presets.SourceLinkedDeployment)
-}
-
-func TestSourceLinkedDeploymentDisabled(t *testing.T) {
-	b, diags := processSourceLinkedBundle(t, false)
-	require.NoError(t, diags.Error())
-	assert.False(t, *b.Config.Presets.SourceLinkedDeployment)
-}
-
-func processSourceLinkedBundle(t *testing.T, presetEnabled bool) (*bundle.Bundle, diag.Diagnostics) {
-	if runtime.GOOS == "windows" {
-		t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
-	}
-
-	b := mockBundle(config.Development)
-
-	workspacePath := "/Workspace/lennart@company.com/"
-	b.SyncRootPath = workspacePath
-	b.Config.Presets.SourceLinkedDeployment = &presetEnabled
-
-	ctx := dbr.MockRuntime(context.Background(), true)
-	m := bundle.Seq(ProcessTargetMode(), ApplyPresets())
-	diags := bundle.Apply(ctx, b, m)
-	return b, diags
-}
--- a/Show More
+++ b/Show More
				`@ -0,0 +1 @@`
				`* @pietern @andrewnester @shreyas-goenka @denik`
				`@ -0,0 +1 @@`
				`$CLI bundle validate -t dev -o json \| jq '{resources, variables}'`
				`@ -0,0 +1 @@`
				`$CLI bundle validate -o json \| jq .workspace`
				`@ -0,0 +1 @@`
				`$CLI bundle validate -o json \| jq .resources.jobs.job1.tags`
				`@ -0,0 +1 @@`
				`BUNDLE_VAR_a=foo BUNDLE_VAR_b=bar $CLI bundle validate -o json \| jq '.variables \| map_values(.value)'`