Merge remote-tracking branch 'origin' into feature/uc-volumes

.codegen/_openapi_sha

@@ -1 +1 @@
-d05898328669a3f8ab0c2ecee37db2673d3ea3f7
+0c86ea6dbd9a730c24ff0d4e509603e476955ac5

.codegen/service.go.tmpl

@@ -5,6 +5,7 @@ package {{(.TrimPrefix "account").SnakeName}}
 import (
     "github.com/databricks/cli/libs/cmdio"
     "github.com/databricks/cli/libs/flags"
+	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/cmd/root"
 	"github.com/databricks/databricks-sdk-go/service/{{.Package.Name}}"
 	"github.com/spf13/cobra"
@@ -231,9 +232,15 @@ func new{{.PascalName}}() *cobra.Command {
 		{{- if .Request }}
 			{{ if .CanUseJson }}
 			if cmd.Flags().Changed("json") {
-					err = {{.CamelName}}Json.Unmarshal(&{{.CamelName}}Req)
+					diags := {{.CamelName}}Json.Unmarshal(&{{.CamelName}}Req)
-					if err != nil {
+					if diags.HasError() {
-						return err
+						return diags.Error()
+					}
+					if len(diags) > 0 {
+						err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags)
+						if err != nil {
+							return err
+						}
 					}
 			}{{end}}{{ if .MustUseJson }}else {
 				return fmt.Errorf("please provide command input in JSON format by specifying the --json flag")

.gitattributes

@@ -6,6 +6,7 @@ cmd/account/cmd.go linguist-generated=true
 cmd/account/credentials/credentials.go linguist-generated=true
 cmd/account/csp-enablement-account/csp-enablement-account.go linguist-generated=true
 cmd/account/custom-app-integration/custom-app-integration.go linguist-generated=true
+cmd/account/disable-legacy-features/disable-legacy-features.go linguist-generated=true
 cmd/account/encryption-keys/encryption-keys.go linguist-generated=true
 cmd/account/esm-enablement-account/esm-enablement-account.go linguist-generated=true
 cmd/account/groups/groups.go linguist-generated=true
@@ -52,6 +53,7 @@ cmd/workspace/dashboard-widgets/dashboard-widgets.go linguist-generated=true
 cmd/workspace/dashboards/dashboards.go linguist-generated=true
 cmd/workspace/data-sources/data-sources.go linguist-generated=true
 cmd/workspace/default-namespace/default-namespace.go linguist-generated=true
+cmd/workspace/disable-legacy-access/disable-legacy-access.go linguist-generated=true
 cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go linguist-generated=true
 cmd/workspace/experiments/experiments.go linguist-generated=true
 cmd/workspace/external-locations/external-locations.go linguist-generated=true
@@ -108,6 +110,7 @@ cmd/workspace/storage-credentials/storage-credentials.go linguist-generated=true
 cmd/workspace/system-schemas/system-schemas.go linguist-generated=true
 cmd/workspace/table-constraints/table-constraints.go linguist-generated=true
 cmd/workspace/tables/tables.go linguist-generated=true
+cmd/workspace/temporary-table-credentials/temporary-table-credentials.go linguist-generated=true
 cmd/workspace/token-management/token-management.go linguist-generated=true
 cmd/workspace/tokens/tokens.go linguist-generated=true
 cmd/workspace/users/users.go linguist-generated=true

.github/workflows/push.yml

@@ -33,7 +33,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.22.x
+          go-version: 1.22.7
 
       - name: Setup Python
         uses: actions/setup-python@v5
@@ -68,7 +68,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.22.x
+          go-version: 1.22.7
 
           # No need to download cached dependencies when running gofmt.
           cache: false
@@ -100,7 +100,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.22.x
+          go-version: 1.22.7
 
       # Github repo: https://github.com/ajv-validator/ajv-cli
       - name: Install ajv-cli

.github/workflows/release-snapshot.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.22.x
+          go-version: 1.22.7
 
           # The default cache key for this action considers only the `go.sum` file.
           # We include .goreleaser.yaml here to differentiate from the cache used by the push action

.github/workflows/release.yml

@@ -22,7 +22,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.22.x
+          go-version: 1.22.7
 
           # The default cache key for this action considers only the `go.sum` file.
           # We include .goreleaser.yaml here to differentiate from the cache used by the push action

CHANGELOG.md

@@ -1,5 +1,100 @@
 # Version changelog
 
+## [Release] Release v0.230.0
+
+Notable changes for Databricks Asset Bundles:
+
+Workspace paths are automatically prefixed with `/Workspace`. In addition, all usage of path strings such as `/Workspace/${workspace.root_path}/...` in bundle configuration is automatically replaced with `${workspace.root_path}/...` and generates a warning as part of bundle validate.
+
+More details can be found here: https://docs.databricks.com/en/release-notes/dev-tools/bundles.html#workspace-paths
+
+Bundles:
+ * Add an error if state files grow bigger than the export limit ([#1795](https://github.com/databricks/cli/pull/1795)).
+ * Always prepend bundle remote paths with /Workspace ([#1724](https://github.com/databricks/cli/pull/1724)).
+ * Add resource path field to bundle workspace configuration ([#1800](https://github.com/databricks/cli/pull/1800)).
+ * Add validation for files with a `.(resource-name).yml` extension ([#1780](https://github.com/databricks/cli/pull/1780)).
+
+Internal:
+ * Remove deprecated or readonly fields from the bundle schema ([#1809](https://github.com/databricks/cli/pull/1809)).
+
+API Changes:
+ * Changed `databricks git-credentials create`, `databricks git-credentials delete`, `databricks git-credentials get`, `databricks git-credentials list`, `databricks git-credentials update` commands .
+ * Changed `databricks repos create`, `databricks repos delete`, `databricks repos get`, `databricks repos update`  command .
+
+OpenAPI commit 0c86ea6dbd9a730c24ff0d4e509603e476955ac5 (2024-10-02)
+Dependency updates:
+ * Upgrade TF provider to 1.53.0 ([#1815](https://github.com/databricks/cli/pull/1815)).
+ * Bump golang.org/x/term from 0.24.0 to 0.25.0 ([#1811](https://github.com/databricks/cli/pull/1811)).
+ * Bump golang.org/x/text from 0.18.0 to 0.19.0 ([#1812](https://github.com/databricks/cli/pull/1812)).
+ * Bump github.com/databricks/databricks-sdk-go from 0.47.0 to 0.48.0 ([#1810](https://github.com/databricks/cli/pull/1810)).
+
+## [Release] Release v0.229.0
+
+Bundles:
+ * Added support for creating all-purpose clusters ([#1698](https://github.com/databricks/cli/pull/1698)).
+ * Reduce time until the prompt is shown for bundle run ([#1727](https://github.com/databricks/cli/pull/1727)).
+ * Use Unity Catalog for pipelines in the default-python template ([#1766](https://github.com/databricks/cli/pull/1766)).
+ * Add verbose flag to the "bundle deploy" command ([#1774](https://github.com/databricks/cli/pull/1774)).
+ * Fixed full variable override detection ([#1787](https://github.com/databricks/cli/pull/1787)).
+ * Add sub-extension to resource files in built-in templates ([#1777](https://github.com/databricks/cli/pull/1777)).
+ * Fix panic in `apply_presets.go` ([#1796](https://github.com/databricks/cli/pull/1796)).
+
+Internal:
+ * Assert tokens are redacted in origin URL when username is not specified ([#1785](https://github.com/databricks/cli/pull/1785)).
+ * Refactor jobs path translation ([#1782](https://github.com/databricks/cli/pull/1782)).
+ * Add JobTaskClusterSpec validate mutator ([#1784](https://github.com/databricks/cli/pull/1784)).
+ * Pin Go toolchain to 1.22.7 ([#1790](https://github.com/databricks/cli/pull/1790)).
+ * Modify SetLocation test utility to take full locations as argument ([#1788](https://github.com/databricks/cli/pull/1788)).
+ * Simplified isFullVariableOverrideDef implementation ([#1791](https://github.com/databricks/cli/pull/1791)).
+ * Sort tasks by `task_key` before generating the Terraform configuration ([#1776](https://github.com/databricks/cli/pull/1776)).
+ * Trim trailing whitespace ([#1794](https://github.com/databricks/cli/pull/1794)).
+ * Move trampoline code into trampoline package ([#1793](https://github.com/databricks/cli/pull/1793)).
+ * Rename `RootPath` -> `BundleRootPath` ([#1792](https://github.com/databricks/cli/pull/1792)).
+
+API Changes:
+ * Changed `databricks apps delete` command to return .
+ * Changed `databricks apps deploy` command with new required argument order.
+ * Changed `databricks apps start` command to return .
+ * Changed `databricks apps stop` command to return .
+ * Added `databricks temporary-table-credentials` command group.
+ * Added `databricks serving-endpoints put-ai-gateway` command.
+ * Added `databricks disable-legacy-access` command group.
+ * Added `databricks account disable-legacy-features` command group.
+
+OpenAPI commit 6f6b1371e640f2dfeba72d365ac566368656f6b6 (2024-09-19)
+Dependency updates:
+ * Upgrade to Go SDK 0.47.0 ([#1799](https://github.com/databricks/cli/pull/1799)).
+ * Upgrade to TF provider 1.52 ([#1781](https://github.com/databricks/cli/pull/1781)).
+ * Bump golang.org/x/mod from 0.20.0 to 0.21.0 ([#1758](https://github.com/databricks/cli/pull/1758)).
+ * Bump github.com/hashicorp/hc-install from 0.7.0 to 0.9.0 ([#1772](https://github.com/databricks/cli/pull/1772)).
+
+## [Release] Release v0.228.1
+
+Bundles:
+ * Added listing cluster filtering for cluster lookups ([#1754](https://github.com/databricks/cli/pull/1754)).
+ * Expand library globs relative to the sync root ([#1756](https://github.com/databricks/cli/pull/1756)).
+ * Fixed generated YAML missing 'default' for empty values ([#1765](https://github.com/databricks/cli/pull/1765)).
+ * Use periodic triggers in all templates ([#1739](https://github.com/databricks/cli/pull/1739)).
+ * Use the friendly name of service principals when shortening their name ([#1770](https://github.com/databricks/cli/pull/1770)).
+ * Fixed detecting full syntax variable override which includes type field ([#1775](https://github.com/databricks/cli/pull/1775)).
+
+Internal:
+ * Pass copy of `dyn.Path` to callback function ([#1747](https://github.com/databricks/cli/pull/1747)).
+ * Make bundle JSON schema modular with `` ([#1700](https://github.com/databricks/cli/pull/1700)).
+ * Alias variables block in the `Target` struct ([#1748](https://github.com/databricks/cli/pull/1748)).
+ * Add end to end integration tests for bundle JSON schema ([#1726](https://github.com/databricks/cli/pull/1726)).
+ * Fix artifact upload integration tests ([#1767](https://github.com/databricks/cli/pull/1767)).
+
+API Changes:
+ * Added `databricks quality-monitors regenerate-dashboard` command.
+
+OpenAPI commit d05898328669a3f8ab0c2ecee37db2673d3ea3f7 (2024-09-04)
+Dependency updates:
+ * Bump golang.org/x/term from 0.23.0 to 0.24.0 ([#1757](https://github.com/databricks/cli/pull/1757)).
+ * Bump golang.org/x/oauth2 from 0.22.0 to 0.23.0 ([#1761](https://github.com/databricks/cli/pull/1761)).
+ * Bump golang.org/x/text from 0.17.0 to 0.18.0 ([#1759](https://github.com/databricks/cli/pull/1759)).
+ * Bump github.com/databricks/databricks-sdk-go from 0.45.0 to 0.46.0 ([#1760](https://github.com/databricks/cli/pull/1760)).
+
 ## [Release] Release v0.228.0
 
 CLI:

bundle/artifacts/expand_globs_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/internal/bundletest"
 	"github.com/databricks/cli/internal/testutil"
+	"github.com/databricks/cli/libs/dyn"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -23,7 +24,7 @@ func TestExpandGlobs_Nominal(t *testing.T) {
 	testutil.Touch(t, tmpDir, "bc.txt")
 
 	b := &bundle.Bundle{
-		RootPath: tmpDir,
+		BundleRootPath: tmpDir,
 		Config: config.Root{
 			Artifacts: config.Artifacts{
 				"test": {
@@ -36,7 +37,7 @@ func TestExpandGlobs_Nominal(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml"))
+	bundletest.SetLocation(b, "artifacts", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}})
 
 	ctx := context.Background()
 	diags := bundle.Apply(ctx, b, bundle.Seq(
@@ -62,7 +63,7 @@ func TestExpandGlobs_InvalidPattern(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	b := &bundle.Bundle{
-		RootPath: tmpDir,
+		BundleRootPath: tmpDir,
 		Config: config.Root{
 			Artifacts: config.Artifacts{
 				"test": {
@@ -77,7 +78,7 @@ func TestExpandGlobs_InvalidPattern(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml"))
+	bundletest.SetLocation(b, "artifacts", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}})
 
 	ctx := context.Background()
 	diags := bundle.Apply(ctx, b, bundle.Seq(
@@ -110,7 +111,7 @@ func TestExpandGlobs_NoMatches(t *testing.T) {
 	testutil.Touch(t, tmpDir, "b2.txt")
 
 	b := &bundle.Bundle{
-		RootPath: tmpDir,
+		BundleRootPath: tmpDir,
 		Config: config.Root{
 			Artifacts: config.Artifacts{
 				"test": {
@@ -125,7 +126,7 @@ func TestExpandGlobs_NoMatches(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml"))
+	bundletest.SetLocation(b, "artifacts", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}})
 
 	ctx := context.Background()
 	diags := bundle.Apply(ctx, b, bundle.Seq(

bundle/artifacts/prepare.go

@@ -47,7 +47,7 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 
 	// If artifact path is not provided, use bundle root dir
 	if artifact.Path == "" {
-		artifact.Path = b.RootPath
+		artifact.Path = b.BundleRootPath
 	}
 
 	if !filepath.IsAbs(artifact.Path) {

bundle/artifacts/whl/autodetect.go

@@ -35,21 +35,21 @@ func (m *detectPkg) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic
 	log.Infof(ctx, "Detecting Python wheel project...")
 
 	// checking if there is setup.py in the bundle root
-	setupPy := filepath.Join(b.RootPath, "setup.py")
+	setupPy := filepath.Join(b.BundleRootPath, "setup.py")
 	_, err := os.Stat(setupPy)
 	if err != nil {
 		log.Infof(ctx, "No Python wheel project found at bundle root folder")
 		return nil
 	}
 
-	log.Infof(ctx, fmt.Sprintf("Found Python wheel project at %s", b.RootPath))
+	log.Infof(ctx, fmt.Sprintf("Found Python wheel project at %s", b.BundleRootPath))
 	module := extractModuleName(setupPy)
 
 	if b.Config.Artifacts == nil {
 		b.Config.Artifacts = make(map[string]*config.Artifact)
 	}
 
-	pkgPath, err := filepath.Abs(b.RootPath)
+	pkgPath, err := filepath.Abs(b.BundleRootPath)
 	if err != nil {
 		return diag.FromErr(err)
 	}

bundle/bundle.go

@@ -31,22 +31,26 @@ import (
 const internalFolder = ".internal"
 
 type Bundle struct {
-	// RootPath contains the directory path to the root of the bundle.
+	// BundleRootPath is the local path to the root directory of the bundle.
 	// It is set when we instantiate a new bundle instance.
-	RootPath string
+	BundleRootPath string
 
-	// BundleRoot is a virtual filesystem path to the root of the bundle.
+	// BundleRoot is a virtual filesystem path to [BundleRootPath].
 	// Exclusively use this field for filesystem operations.
 	BundleRoot vfs.Path
 
-	// SyncRoot is a virtual filesystem path to the root directory of the files that are synchronized to the workspace.
-	// It can be an ancestor to [BundleRoot], but not a descendant; that is, [SyncRoot] must contain [BundleRoot].
-	SyncRoot vfs.Path
-
 	// SyncRootPath is the local path to the root directory of files that are synchronized to the workspace.
-	// It is equal to `SyncRoot.Native()` and included as dedicated field for convenient access.
+	// By default, it is the same as [BundleRootPath].
+	// If it is different, it must be an ancestor to [BundleRootPath].
+	// That is, [SyncRootPath] must contain [BundleRootPath].
 	SyncRootPath string
 
+	// SyncRoot is a virtual filesystem path to [SyncRootPath].
+	// Exclusively use this field for filesystem operations.
+	SyncRoot vfs.Path
+
+	// Config contains the bundle configuration.
+	// It is loaded from the bundle configuration files and mutators may update it.
 	Config config.Root
 
 	// Metadata about the bundle deployment. This is the interface Databricks services
@@ -84,14 +88,14 @@ type Bundle struct {
 
 func Load(ctx context.Context, path string) (*Bundle, error) {
 	b := &Bundle{
-		RootPath:   filepath.Clean(path),
+		BundleRootPath: filepath.Clean(path),
-		BundleRoot: vfs.MustNew(path),
+		BundleRoot:     vfs.MustNew(path),
 	}
 	configFile, err := config.FileNames.FindInPath(path)
 	if err != nil {
 		return nil, err
 	}
-	log.Debugf(ctx, "Found bundle root at %s (file %s)", b.RootPath, configFile)
+	log.Debugf(ctx, "Found bundle root at %s (file %s)", b.BundleRootPath, configFile)
 	return b, nil
 }
 
@@ -160,7 +164,7 @@ func (b *Bundle) CacheDir(ctx context.Context, paths ...string) (string, error)
 	if !exists || cacheDirName == "" {
 		cacheDirName = filepath.Join(
 			// Anchor at bundle root directory.
-			b.RootPath,
+			b.BundleRootPath,
 			// Static cache directory.
 			".databricks",
 			"bundle",
@@ -212,7 +216,7 @@ func (b *Bundle) GetSyncIncludePatterns(ctx context.Context) ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
-	internalDirRel, err := filepath.Rel(b.RootPath, internalDir)
+	internalDirRel, err := filepath.Rel(b.BundleRootPath, internalDir)
 	if err != nil {
 		return nil, err
 	}

bundle/bundle_read_only.go

@@ -21,7 +21,7 @@ func (r ReadOnlyBundle) Config() config.Root {
 }
 
 func (r ReadOnlyBundle) RootPath() string {
-	return r.b.RootPath
+	return r.b.BundleRootPath
 }
 
 func (r ReadOnlyBundle) BundleRoot() vfs.Path {

bundle/bundle_test.go

@@ -79,7 +79,7 @@ func TestBundleMustLoadSuccess(t *testing.T) {
 	t.Setenv(env.RootVariable, "./tests/basic")
 	b, err := MustLoad(context.Background())
 	require.NoError(t, err)
-	assert.Equal(t, "tests/basic", filepath.ToSlash(b.RootPath))
+	assert.Equal(t, "tests/basic", filepath.ToSlash(b.BundleRootPath))
 }
 
 func TestBundleMustLoadFailureWithEnv(t *testing.T) {
@@ -98,7 +98,7 @@ func TestBundleTryLoadSuccess(t *testing.T) {
 	t.Setenv(env.RootVariable, "./tests/basic")
 	b, err := TryLoad(context.Background())
 	require.NoError(t, err)
-	assert.Equal(t, "tests/basic", filepath.ToSlash(b.RootPath))
+	assert.Equal(t, "tests/basic", filepath.ToSlash(b.BundleRootPath))
 }
 
 func TestBundleTryLoadFailureWithEnv(t *testing.T) {

bundle/config/bundle.go

@@ -38,8 +38,11 @@ type Bundle struct {
 	// Annotated readonly as this should be set at the target level.
 	Mode Mode `json:"mode,omitempty" bundle:"readonly"`
 
-	// Overrides the compute used for jobs and other supported assets.
+	// DEPRECATED: Overrides the compute used for jobs and other supported assets.
-	ComputeID string `json:"compute_id,omitempty"`
+	ComputeId string `json:"compute_id,omitempty"`
+
+	// Overrides the cluster used for jobs and other supported assets.
+	ClusterId string `json:"cluster_id,omitempty"`
 
 	// Deployment section specifies deployment related configuration for bundle
 	Deployment Deployment `json:"deployment,omitempty"`

bundle/config/loader/entry_point.go

@@ -20,7 +20,7 @@ func (m *entryPoint) Name() string {
 }
 
 func (m *entryPoint) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
-	path, err := config.FileNames.FindInPath(b.RootPath)
+	path, err := config.FileNames.FindInPath(b.BundleRootPath)
 	if err != nil {
 		return diag.FromErr(err)
 	}

bundle/config/loader/entry_point_test.go

@@ -18,7 +18,7 @@ func TestEntryPointNoRootPath(t *testing.T) {
 
 func TestEntryPoint(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: "testdata",
+		BundleRootPath: "testdata/basic",
 	}
 	diags := bundle.Apply(context.Background(), b, loader.EntryPoint())
 	require.NoError(t, diags.Error())

bundle/config/loader/process_include.go

@@ -3,12 +3,135 @@ package loader
 import (
 	"context"
 	"fmt"
+	"slices"
+	"sort"
+	"strings"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
 )
 
+func validateFileFormat(configRoot dyn.Value, filePath string) diag.Diagnostics {
+	for _, resourceDescription := range config.SupportedResources() {
+		singularName := resourceDescription.SingularName
+
+		for _, yamlExt := range []string{"yml", "yaml"} {
+			ext := fmt.Sprintf(".%s.%s", singularName, yamlExt)
+			if strings.HasSuffix(filePath, ext) {
+				return validateSingleResourceDefined(configRoot, ext, singularName)
+			}
+		}
+	}
+
+	return nil
+}
+
+func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.Diagnostics {
+	type resource struct {
+		path  dyn.Path
+		value dyn.Value
+		typ   string
+		key   string
+	}
+
+	resources := []resource{}
+	supportedResources := config.SupportedResources()
+
+	// Gather all resources defined in the resources block.
+	_, err := dyn.MapByPattern(
+		configRoot,
+		dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()),
+		func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+			// The key for the resource, e.g. "my_job" for jobs.my_job.
+			k := p[2].Key()
+			// The type of the resource, e.g. "job" for jobs.my_job.
+			typ := supportedResources[p[1].Key()].SingularName
+
+			resources = append(resources, resource{path: p, value: v, typ: typ, key: k})
+			return v, nil
+		})
+	if err != nil {
+		return diag.FromErr(err)
+	}
+
+	// Gather all resources defined in a target block.
+	_, err = dyn.MapByPattern(
+		configRoot,
+		dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey(), dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()),
+		func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+			// The key for the resource, e.g. "my_job" for jobs.my_job.
+			k := p[4].Key()
+			// The type of the resource, e.g. "job" for jobs.my_job.
+			typ := supportedResources[p[3].Key()].SingularName
+
+			resources = append(resources, resource{path: p, value: v, typ: typ, key: k})
+			return v, nil
+		})
+	if err != nil {
+		return diag.FromErr(err)
+	}
+
+	typeMatch := true
+	seenKeys := map[string]struct{}{}
+	for _, rr := range resources {
+		// case: The resource is not of the correct type.
+		if rr.typ != typ {
+			typeMatch = false
+			break
+		}
+
+		seenKeys[rr.key] = struct{}{}
+	}
+
+	// Format matches. There's at most one resource defined in the file.
+	// The resource is also of the correct type.
+	if typeMatch && len(seenKeys) <= 1 {
+		return nil
+	}
+
+	detail := strings.Builder{}
+	detail.WriteString("The following resources are defined or configured in this file:\n")
+	lines := []string{}
+	for _, r := range resources {
+		lines = append(lines, fmt.Sprintf("  - %s (%s)\n", r.key, r.typ))
+	}
+	// Sort the lines to print to make the output deterministic.
+	sort.Strings(lines)
+	// Compact the lines before writing them to the message to remove any duplicate lines.
+	// This is needed because we do not dedup earlier when gathering the resources
+	// and it's valid to define the same resource in both the resources and targets block.
+	lines = slices.Compact(lines)
+	for _, l := range lines {
+		detail.WriteString(l)
+	}
+
+	locations := []dyn.Location{}
+	paths := []dyn.Path{}
+	for _, rr := range resources {
+		locations = append(locations, rr.value.Locations()...)
+		paths = append(paths, rr.path)
+	}
+	// Sort the locations and paths to make the output deterministic.
+	sort.Slice(locations, func(i, j int) bool {
+		return locations[i].String() < locations[j].String()
+	})
+	sort.Slice(paths, func(i, j int) bool {
+		return paths[i].String() < paths[j].String()
+	})
+
+	return diag.Diagnostics{
+		{
+			Severity:  diag.Recommendation,
+			Summary:   fmt.Sprintf("define a single %s in a file with the %s extension.", strings.ReplaceAll(typ, "_", " "), ext),
+			Detail:    detail.String(),
+			Locations: locations,
+			Paths:     paths,
+		},
+	}
+}
+
 type processInclude struct {
 	fullPath string
 	relPath  string
@@ -31,6 +154,13 @@ func (m *processInclude) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos
 	if diags.HasError() {
 		return diags
 	}
+
+	// Add any diagnostics associated with the file format.
+	diags = append(diags, validateFileFormat(this.Value(), m.relPath)...)
+	if diags.HasError() {
+		return diags
+	}
+
 	err := b.Config.Merge(this)
 	if err != nil {
 		diags = diags.Extend(diag.FromErr(err))

bundle/config/loader/process_include_test.go

@@ -8,13 +8,15 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/loader"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestProcessInclude(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: "testdata",
+		BundleRootPath: "testdata/basic",
 		Config: config.Root{
 			Workspace: config.Workspace{
 				Host: "foo",
@@ -22,7 +24,7 @@ func TestProcessInclude(t *testing.T) {
 		},
 	}
 
-	m := loader.ProcessInclude(filepath.Join(b.RootPath, "host.yml"), "host.yml")
+	m := loader.ProcessInclude(filepath.Join(b.BundleRootPath, "host.yml"), "host.yml")
 	assert.Equal(t, "ProcessInclude(host.yml)", m.Name())
 
 	// Assert the host value prior to applying the mutator
@@ -33,3 +35,184 @@ func TestProcessInclude(t *testing.T) {
 	require.NoError(t, diags.Error())
 	assert.Equal(t, "bar", b.Config.Workspace.Host)
 }
+
+func TestProcessIncludeFormatMatch(t *testing.T) {
+	for _, fileName := range []string{
+		"one_job.job.yml",
+		"one_pipeline.pipeline.yaml",
+		"two_job.yml",
+		"job_and_pipeline.yml",
+		"multiple_resources.yml",
+	} {
+		t.Run(fileName, func(t *testing.T) {
+			b := &bundle.Bundle{
+				BundleRootPath: "testdata/format_match",
+				Config: config.Root{
+					Bundle: config.Bundle{
+						Name: "format_test",
+					},
+				},
+			}
+
+			m := loader.ProcessInclude(filepath.Join(b.BundleRootPath, fileName), fileName)
+			diags := bundle.Apply(context.Background(), b, m)
+			assert.Empty(t, diags)
+		})
+	}
+}
+
+func TestProcessIncludeFormatNotMatch(t *testing.T) {
+	for fileName, expectedDiags := range map[string]diag.Diagnostics{
+		"single_job.pipeline.yaml": {
+			{
+				Severity: diag.Recommendation,
+				Summary:  "define a single pipeline in a file with the .pipeline.yaml extension.",
+				Detail:   "The following resources are defined or configured in this file:\n  - job1 (job)\n",
+				Locations: []dyn.Location{
+					{File: filepath.FromSlash("testdata/format_not_match/single_job.pipeline.yaml"), Line: 11, Column: 11},
+					{File: filepath.FromSlash("testdata/format_not_match/single_job.pipeline.yaml"), Line: 4, Column: 7},
+				},
+				Paths: []dyn.Path{
+					dyn.MustPathFromString("resources.jobs.job1"),
+					dyn.MustPathFromString("targets.target1.resources.jobs.job1"),
+				},
+			},
+		},
+		"job_and_pipeline.job.yml": {
+			{
+				Severity: diag.Recommendation,
+				Summary:  "define a single job in a file with the .job.yml extension.",
+				Detail:   "The following resources are defined or configured in this file:\n  - job1 (job)\n  - pipeline1 (pipeline)\n",
+				Locations: []dyn.Location{
+					{File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.job.yml"), Line: 11, Column: 11},
+					{File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.job.yml"), Line: 4, Column: 7},
+				},
+				Paths: []dyn.Path{
+					dyn.MustPathFromString("resources.pipelines.pipeline1"),
+					dyn.MustPathFromString("targets.target1.resources.jobs.job1"),
+				},
+			},
+		},
+		"job_and_pipeline.experiment.yml": {
+			{
+				Severity: diag.Recommendation,
+				Summary:  "define a single experiment in a file with the .experiment.yml extension.",
+				Detail:   "The following resources are defined or configured in this file:\n  - job1 (job)\n  - pipeline1 (pipeline)\n",
+				Locations: []dyn.Location{
+					{File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.experiment.yml"), Line: 11, Column: 11},
+					{File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.experiment.yml"), Line: 4, Column: 7},
+				},
+				Paths: []dyn.Path{
+					dyn.MustPathFromString("resources.pipelines.pipeline1"),
+					dyn.MustPathFromString("targets.target1.resources.jobs.job1"),
+				},
+			},
+		},
+		"two_jobs.job.yml": {
+			{
+				Severity: diag.Recommendation,
+				Summary:  "define a single job in a file with the .job.yml extension.",
+				Detail:   "The following resources are defined or configured in this file:\n  - job1 (job)\n  - job2 (job)\n",
+				Locations: []dyn.Location{
+					{File: filepath.FromSlash("testdata/format_not_match/two_jobs.job.yml"), Line: 4, Column: 7},
+					{File: filepath.FromSlash("testdata/format_not_match/two_jobs.job.yml"), Line: 7, Column: 7},
+				},
+				Paths: []dyn.Path{
+					dyn.MustPathFromString("resources.jobs.job1"),
+					dyn.MustPathFromString("resources.jobs.job2"),
+				},
+			},
+		},
+		"second_job_in_target.job.yml": {
+			{
+				Severity: diag.Recommendation,
+				Summary:  "define a single job in a file with the .job.yml extension.",
+				Detail:   "The following resources are defined or configured in this file:\n  - job1 (job)\n  - job2 (job)\n",
+				Locations: []dyn.Location{
+					{File: filepath.FromSlash("testdata/format_not_match/second_job_in_target.job.yml"), Line: 11, Column: 11},
+					{File: filepath.FromSlash("testdata/format_not_match/second_job_in_target.job.yml"), Line: 4, Column: 7},
+				},
+				Paths: []dyn.Path{
+					dyn.MustPathFromString("resources.jobs.job1"),
+					dyn.MustPathFromString("targets.target1.resources.jobs.job2"),
+				},
+			},
+		},
+		"two_jobs_in_target.job.yml": {
+			{
+				Severity: diag.Recommendation,
+				Summary:  "define a single job in a file with the .job.yml extension.",
+				Detail:   "The following resources are defined or configured in this file:\n  - job1 (job)\n  - job2 (job)\n",
+				Locations: []dyn.Location{
+					{File: filepath.FromSlash("testdata/format_not_match/two_jobs_in_target.job.yml"), Line: 6, Column: 11},
+					{File: filepath.FromSlash("testdata/format_not_match/two_jobs_in_target.job.yml"), Line: 8, Column: 11},
+				},
+				Paths: []dyn.Path{
+					dyn.MustPathFromString("targets.target1.resources.jobs.job1"),
+					dyn.MustPathFromString("targets.target1.resources.jobs.job2"),
+				},
+			},
+		},
+		"multiple_resources.model_serving_endpoint.yml": {
+			{
+				Severity: diag.Recommendation,
+				Summary:  "define a single model serving endpoint in a file with the .model_serving_endpoint.yml extension.",
+				Detail: `The following resources are defined or configured in this file:
+  - experiment1 (experiment)
+  - job1 (job)
+  - job2 (job)
+  - job3 (job)
+  - model1 (model)
+  - model_serving_endpoint1 (model_serving_endpoint)
+  - pipeline1 (pipeline)
+  - pipeline2 (pipeline)
+  - quality_monitor1 (quality_monitor)
+  - registered_model1 (registered_model)
+  - schema1 (schema)
+`,
+				Locations: []dyn.Location{
+					{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 12, Column: 7},
+					{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 14, Column: 7},
+					{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 18, Column: 7},
+					{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 22, Column: 7},
+					{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 24, Column: 7},
+					{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 28, Column: 7},
+					{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 35, Column: 11},
+					{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 39, Column: 11},
+					{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 43, Column: 11},
+					{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 4, Column: 7},
+					{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 8, Column: 7},
+				},
+				Paths: []dyn.Path{
+					dyn.MustPathFromString("resources.experiments.experiment1"),
+					dyn.MustPathFromString("resources.jobs.job1"),
+					dyn.MustPathFromString("resources.jobs.job2"),
+					dyn.MustPathFromString("resources.model_serving_endpoints.model_serving_endpoint1"),
+					dyn.MustPathFromString("resources.models.model1"),
+					dyn.MustPathFromString("resources.pipelines.pipeline1"),
+					dyn.MustPathFromString("resources.pipelines.pipeline2"),
+					dyn.MustPathFromString("resources.schemas.schema1"),
+					dyn.MustPathFromString("targets.target1.resources.jobs.job3"),
+					dyn.MustPathFromString("targets.target1.resources.quality_monitors.quality_monitor1"),
+					dyn.MustPathFromString("targets.target1.resources.registered_models.registered_model1"),
+				},
+			},
+		},
+	} {
+		t.Run(fileName, func(t *testing.T) {
+			b := &bundle.Bundle{
+				BundleRootPath: "testdata/format_not_match",
+				Config: config.Root{
+					Bundle: config.Bundle{
+						Name: "format_test",
+					},
+				},
+			}
+
+			m := loader.ProcessInclude(filepath.Join(b.BundleRootPath, fileName), fileName)
+			diags := bundle.Apply(context.Background(), b, m)
+			require.Len(t, diags, 1)
+			assert.Equal(t, expectedDiags, diags)
+		})
+	}
+}

bundle/config/loader/process_root_includes.go

@@ -47,7 +47,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.
 		}
 
 		// Anchor includes to the bundle root path.
-		matches, err := filepath.Glob(filepath.Join(b.RootPath, entry))
+		matches, err := filepath.Glob(filepath.Join(b.BundleRootPath, entry))
 		if err != nil {
 			return diag.FromErr(err)
 		}
@@ -61,7 +61,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.
 		// Filter matches to ones we haven't seen yet.
 		var includes []string
 		for _, match := range matches {
-			rel, err := filepath.Rel(b.RootPath, match)
+			rel, err := filepath.Rel(b.BundleRootPath, match)
 			if err != nil {
 				return diag.FromErr(err)
 			}
@@ -76,7 +76,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.
 		slices.Sort(includes)
 		files = append(files, includes...)
 		for _, include := range includes {
-			out = append(out, ProcessInclude(filepath.Join(b.RootPath, include), include))
+			out = append(out, ProcessInclude(filepath.Join(b.BundleRootPath, include), include))
 		}
 	}
 

bundle/config/loader/process_root_includes_test.go

@@ -15,7 +15,7 @@ import (
 
 func TestProcessRootIncludesEmpty(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: ".",
+		BundleRootPath: ".",
 	}
 	diags := bundle.Apply(context.Background(), b, loader.ProcessRootIncludes())
 	require.NoError(t, diags.Error())
@@ -30,7 +30,7 @@ func TestProcessRootIncludesAbs(t *testing.T) {
 	}
 
 	b := &bundle.Bundle{
-		RootPath: ".",
+		BundleRootPath: ".",
 		Config: config.Root{
 			Include: []string{
 				"/tmp/*.yml",
@@ -44,7 +44,7 @@ func TestProcessRootIncludesAbs(t *testing.T) {
 
 func TestProcessRootIncludesSingleGlob(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Include: []string{
 				"*.yml",
@@ -52,9 +52,9 @@ func TestProcessRootIncludesSingleGlob(t *testing.T) {
 		},
 	}
 
-	testutil.Touch(t, b.RootPath, "databricks.yml")
+	testutil.Touch(t, b.BundleRootPath, "databricks.yml")
-	testutil.Touch(t, b.RootPath, "a.yml")
+	testutil.Touch(t, b.BundleRootPath, "a.yml")
-	testutil.Touch(t, b.RootPath, "b.yml")
+	testutil.Touch(t, b.BundleRootPath, "b.yml")
 
 	diags := bundle.Apply(context.Background(), b, loader.ProcessRootIncludes())
 	require.NoError(t, diags.Error())
@@ -63,7 +63,7 @@ func TestProcessRootIncludesSingleGlob(t *testing.T) {
 
 func TestProcessRootIncludesMultiGlob(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Include: []string{
 				"a*.yml",
@@ -72,8 +72,8 @@ func TestProcessRootIncludesMultiGlob(t *testing.T) {
 		},
 	}
 
-	testutil.Touch(t, b.RootPath, "a1.yml")
+	testutil.Touch(t, b.BundleRootPath, "a1.yml")
-	testutil.Touch(t, b.RootPath, "b1.yml")
+	testutil.Touch(t, b.BundleRootPath, "b1.yml")
 
 	diags := bundle.Apply(context.Background(), b, loader.ProcessRootIncludes())
 	require.NoError(t, diags.Error())
@@ -82,7 +82,7 @@ func TestProcessRootIncludesMultiGlob(t *testing.T) {
 
 func TestProcessRootIncludesRemoveDups(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Include: []string{
 				"*.yml",
@@ -91,7 +91,7 @@ func TestProcessRootIncludesRemoveDups(t *testing.T) {
 		},
 	}
 
-	testutil.Touch(t, b.RootPath, "a.yml")
+	testutil.Touch(t, b.BundleRootPath, "a.yml")
 
 	diags := bundle.Apply(context.Background(), b, loader.ProcessRootIncludes())
 	require.NoError(t, diags.Error())
@@ -100,7 +100,7 @@ func TestProcessRootIncludesRemoveDups(t *testing.T) {
 
 func TestProcessRootIncludesNotExists(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Include: []string{
 				"notexist.yml",

bundle/config/loader/testdata/format_match/job_and_pipeline.yml

@@ -0,0 +1,11 @@
+resources:
+  pipelines:
+    pipeline1:
+      name: pipeline1
+
+targets:
+  target1:
+    resources:
+      jobs:
+        job1:
+          name: job1

bundle/config/loader/testdata/format_match/multiple_resources.yml

@@ -0,0 +1,43 @@
+resources:
+  experiments:
+    experiment1:
+      name: experiment1
+
+  model_serving_endpoints:
+    model_serving_endpoint1:
+      name: model_serving_endpoint1
+
+  jobs:
+    job1:
+      name: job1
+    job2:
+      name: job2
+
+  models:
+    model1:
+      name: model1
+
+  pipelines:
+    pipeline1:
+      name: pipeline1
+    pipeline2:
+      name: pipeline2
+
+  schemas:
+    schema1:
+      name: schema1
+
+targets:
+  target1:
+    resources:
+      quality_monitors:
+        quality_monitor1:
+          baseline_table_name: quality_monitor1
+
+      jobs:
+        job3:
+          name: job3
+
+      registered_models:
+        registered_model1:
+          name: registered_model1

bundle/config/loader/testdata/format_match/one_job.job.yml

@@ -0,0 +1,11 @@
+resources:
+  jobs:
+    job1:
+      name: job1
+
+targets:
+  target1:
+    resources:
+      jobs:
+        job1:
+          description: job1

bundle/config/loader/testdata/format_match/one_pipeline.pipeline.yaml

@@ -0,0 +1,4 @@
+resources:
+  pipelines:
+    pipeline1:
+      name: pipeline1

bundle/config/loader/testdata/format_match/two_job.yml

@@ -0,0 +1,7 @@
+resources:
+  jobs:
+    job1:
+      name: job1
+
+    job2:
+      name: job2

bundle/config/loader/testdata/format_not_match/job_and_pipeline.experiment.yml

@@ -0,0 +1,11 @@
+resources:
+  pipelines:
+    pipeline1:
+      name: pipeline1
+
+targets:
+  target1:
+    resources:
+      jobs:
+        job1:
+          name: job1

bundle/config/loader/testdata/format_not_match/job_and_pipeline.job.yml

@@ -0,0 +1,11 @@
+resources:
+  pipelines:
+    pipeline1:
+      name: pipeline1
+
+targets:
+  target1:
+    resources:
+      jobs:
+        job1:
+          name: job1

bundle/config/loader/testdata/format_not_match/multiple_resources.model_serving_endpoint.yml

@@ -0,0 +1,43 @@
+resources:
+  experiments:
+    experiment1:
+      name: experiment1
+
+  model_serving_endpoints:
+    model_serving_endpoint1:
+      name: model_serving_endpoint1
+
+  jobs:
+    job1:
+      name: job1
+    job2:
+      name: job2
+
+  models:
+    model1:
+      name: model1
+
+  pipelines:
+    pipeline1:
+      name: pipeline1
+    pipeline2:
+      name: pipeline2
+
+  schemas:
+    schema1:
+      name: schema1
+
+targets:
+  target1:
+    resources:
+      quality_monitors:
+        quality_monitor1:
+          baseline_table_name: quality_monitor1
+
+      jobs:
+        job3:
+          name: job3
+
+      registered_models:
+        registered_model1:
+          name: registered_model1

bundle/config/loader/testdata/format_not_match/second_job_in_target.job.yml

@@ -0,0 +1,11 @@
+resources:
+  jobs:
+    job1:
+      name: job1
+
+targets:
+  target1:
+    resources:
+      jobs:
+        job2:
+          name: job2

bundle/config/loader/testdata/format_not_match/single_job.pipeline.yaml

@@ -0,0 +1,11 @@
+resources:
+  jobs:
+    job1:
+      name: job1
+
+targets:
+  target1:
+    resources:
+      jobs:
+        job1:
+          description: job1

bundle/config/loader/testdata/format_not_match/two_jobs.job.yml

@@ -0,0 +1,7 @@
+resources:
+  jobs:
+    job1:
+      name: job1
+
+    job2:
+      name: job2

bundle/config/loader/testdata/format_not_match/two_jobs_in_target.job.yml

@@ -0,0 +1,8 @@
+targets:
+  target1:
+    resources:
+      jobs:
+        job1:
+          description: job1
+        job2:
+          description: job2

bundle/config/mutator/apply_presets.go

@@ -35,8 +35,10 @@ func (m *applyPresets) Name() string {
 }
 
 func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	var diags diag.Diagnostics
+
 	if d := validatePauseStatus(b); d != nil {
-		return d
+		diags = diags.Extend(d)
 	}
 
 	r := b.Config.Resources
@@ -45,7 +47,11 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 	tags := toTagArray(t.Tags)
 
 	// Jobs presets: Prefix, Tags, JobsMaxConcurrentRuns, TriggerPauseStatus
-	for _, j := range r.Jobs {
+	for key, j := range r.Jobs {
+		if j.JobSettings == nil {
+			diags = diags.Extend(diag.Errorf("job %s is not defined", key))
+			continue
+		}
 		j.Name = prefix + j.Name
 		if j.Tags == nil {
 			j.Tags = make(map[string]string)
@@ -77,20 +83,27 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 	}
 
 	// Pipelines presets: Prefix, PipelinesDevelopment
-	for i := range r.Pipelines {
+	for key, p := range r.Pipelines {
-		r.Pipelines[i].Name = prefix + r.Pipelines[i].Name
+		if p.PipelineSpec == nil {
+			diags = diags.Extend(diag.Errorf("pipeline %s is not defined", key))
+			continue
+		}
+		p.Name = prefix + p.Name
 		if config.IsExplicitlyEnabled(t.PipelinesDevelopment) {
-			r.Pipelines[i].Development = true
+			p.Development = true
 		}
 		if t.TriggerPauseStatus == config.Paused {
-			r.Pipelines[i].Continuous = false
+			p.Continuous = false
 		}
-
 		// As of 2024-06, pipelines don't yet support tags
 	}
 
 	// Models presets: Prefix, Tags
-	for _, m := range r.Models {
+	for key, m := range r.Models {
+		if m.Model == nil {
+			diags = diags.Extend(diag.Errorf("model %s is not defined", key))
+			continue
+		}
 		m.Name = prefix + m.Name
 		for _, t := range tags {
 			exists := slices.ContainsFunc(m.Tags, func(modelTag ml.ModelTag) bool {
@@ -104,7 +117,11 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 	}
 
 	// Experiments presets: Prefix, Tags
-	for _, e := range r.Experiments {
+	for key, e := range r.Experiments {
+		if e.Experiment == nil {
+			diags = diags.Extend(diag.Errorf("experiment %s is not defined", key))
+			continue
+		}
 		filepath := e.Name
 		dir := path.Dir(filepath)
 		base := path.Base(filepath)
@@ -128,34 +145,50 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 	}
 
 	// Model serving endpoint presets: Prefix
-	for i := range r.ModelServingEndpoints {
+	for key, e := range r.ModelServingEndpoints {
-		r.ModelServingEndpoints[i].Name = normalizePrefix(prefix) + r.ModelServingEndpoints[i].Name
+		if e.CreateServingEndpoint == nil {
+			diags = diags.Extend(diag.Errorf("model serving endpoint %s is not defined", key))
+			continue
+		}
+		e.Name = normalizePrefix(prefix) + e.Name
 
 		// As of 2024-06, model serving endpoints don't yet support tags
 	}
 
 	// Registered models presets: Prefix
-	for i := range r.RegisteredModels {
+	for key, m := range r.RegisteredModels {
-		r.RegisteredModels[i].Name = normalizePrefix(prefix) + r.RegisteredModels[i].Name
+		if m.CreateRegisteredModelRequest == nil {
+			diags = diags.Extend(diag.Errorf("registered model %s is not defined", key))
+			continue
+		}
+		m.Name = normalizePrefix(prefix) + m.Name
 
 		// As of 2024-06, registered models don't yet support tags
 	}
 
-	// Quality monitors presets: Prefix
+	// Quality monitors presets: Schedule
 	if t.TriggerPauseStatus == config.Paused {
-		for i := range r.QualityMonitors {
+		for key, q := range r.QualityMonitors {
+			if q.CreateMonitor == nil {
+				diags = diags.Extend(diag.Errorf("quality monitor %s is not defined", key))
+				continue
+			}
 			// Remove all schedules from monitors, since they don't support pausing/unpausing.
 			// Quality monitors might support the "pause" property in the future, so at the
 			// CLI level we do respect that property if it is set to "unpaused."
-			if r.QualityMonitors[i].Schedule != nil && r.QualityMonitors[i].Schedule.PauseStatus != catalog.MonitorCronSchedulePauseStatusUnpaused {
+			if q.Schedule != nil && q.Schedule.PauseStatus != catalog.MonitorCronSchedulePauseStatusUnpaused {
-				r.QualityMonitors[i].Schedule = nil
+				q.Schedule = nil
 			}
 		}
 	}
 
 	// Schemas: Prefix
-	for i := range r.Schemas {
+	for key, s := range r.Schemas {
-		r.Schemas[i].Name = normalizePrefix(prefix) + r.Schemas[i].Name
+		if s.CreateSchema == nil {
+			diags = diags.Extend(diag.Errorf("schema %s is not defined", key))
+			continue
+		}
+		s.Name = normalizePrefix(prefix) + s.Name
 		// HTTP API for schemas doesn't yet support tags. It's only supported in
 		// the Databricks UI and via the SQL API.
 	}
@@ -167,7 +200,26 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 		// the Databricks UI and via the SQL API.
 	}
 
-	return nil
+	// Clusters: Prefix, Tags
+	for key, c := range r.Clusters {
+		if c.ClusterSpec == nil {
+			diags = diags.Extend(diag.Errorf("cluster %s is not defined", key))
+			continue
+		}
+		c.ClusterName = prefix + c.ClusterName
+		if c.CustomTags == nil {
+			c.CustomTags = make(map[string]string)
+		}
+		for _, tag := range tags {
+			normalisedKey := b.Tagging.NormalizeKey(tag.Key)
+			normalisedValue := b.Tagging.NormalizeValue(tag.Value)
+			if _, ok := c.CustomTags[normalisedKey]; !ok {
+				c.CustomTags[normalisedKey] = normalisedValue
+			}
+		}
+	}
+
+	return diags
 }
 
 func validatePauseStatus(b *bundle.Bundle) diag.Diagnostics {

bundle/config/mutator/apply_presets_test.go

@@ -307,3 +307,116 @@ func TestApplyPresetsJobsMaxConcurrentRuns(t *testing.T) {
 		})
 	}
 }
+
+func TestApplyPresetsPrefixWithoutJobSettings(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"job1": {}, // no jobsettings inside
+				},
+			},
+			Presets: config.Presets{
+				NamePrefix: "prefix-",
+			},
+		},
+	}
+
+	ctx := context.Background()
+	diags := bundle.Apply(ctx, b, mutator.ApplyPresets())
+
+	require.ErrorContains(t, diags.Error(), "job job1 is not defined")
+}
+
+func TestApplyPresetsResourceNotDefined(t *testing.T) {
+	tests := []struct {
+		resources config.Resources
+		error     string
+	}{
+		{
+			resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"job1": {}, // no jobsettings inside
+				},
+			},
+			error: "job job1 is not defined",
+		},
+		{
+			resources: config.Resources{
+				Pipelines: map[string]*resources.Pipeline{
+					"pipeline1": {}, // no pipelinespec inside
+				},
+			},
+			error: "pipeline pipeline1 is not defined",
+		},
+		{
+			resources: config.Resources{
+				Models: map[string]*resources.MlflowModel{
+					"model1": {}, // no model inside
+				},
+			},
+			error: "model model1 is not defined",
+		},
+		{
+			resources: config.Resources{
+				Experiments: map[string]*resources.MlflowExperiment{
+					"experiment1": {}, // no experiment inside
+				},
+			},
+			error: "experiment experiment1 is not defined",
+		},
+		{
+			resources: config.Resources{
+				ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{
+					"endpoint1": {}, // no CreateServingEndpoint inside
+				},
+				RegisteredModels: map[string]*resources.RegisteredModel{
+					"model1": {}, // no CreateRegisteredModelRequest inside
+				},
+			},
+			error: "model serving endpoint endpoint1 is not defined",
+		},
+		{
+			resources: config.Resources{
+				QualityMonitors: map[string]*resources.QualityMonitor{
+					"monitor1": {}, // no CreateMonitor inside
+				},
+			},
+			error: "quality monitor monitor1 is not defined",
+		},
+		{
+			resources: config.Resources{
+				Schemas: map[string]*resources.Schema{
+					"schema1": {}, // no CreateSchema inside
+				},
+			},
+			error: "schema schema1 is not defined",
+		},
+		{
+			resources: config.Resources{
+				Clusters: map[string]*resources.Cluster{
+					"cluster1": {}, // no ClusterSpec inside
+				},
+			},
+			error: "cluster cluster1 is not defined",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.error, func(t *testing.T) {
+			b := &bundle.Bundle{
+				Config: config.Root{
+					Resources: tt.resources,
+					Presets: config.Presets{
+						TriggerPauseStatus: config.Paused,
+					},
+				},
+			}
+
+			ctx := context.Background()
+			diags := bundle.Apply(ctx, b, mutator.ApplyPresets())
+
+			require.ErrorContains(t, diags.Error(), tt.error)
+		})
+	}
+}

bundle/config/mutator/compute_id_compat.go

@@ -0,0 +1,87 @@
+package mutator
+
+import (
+	"context"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+)
+
+type computeIdToClusterId struct{}
+
+func ComputeIdToClusterId() bundle.Mutator {
+	return &computeIdToClusterId{}
+}
+
+func (m *computeIdToClusterId) Name() string {
+	return "ComputeIdToClusterId"
+}
+
+func (m *computeIdToClusterId) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	var diags diag.Diagnostics
+
+	// The "compute_id" key is set; rewrite it to "cluster_id".
+	err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
+		v, d := rewriteComputeIdToClusterId(v, dyn.NewPath(dyn.Key("bundle")))
+		diags = diags.Extend(d)
+
+		// Check if the "compute_id" key is set in any target overrides.
+		return dyn.MapByPattern(v, dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+			v, d := rewriteComputeIdToClusterId(v, dyn.Path{})
+			diags = diags.Extend(d)
+			return v, nil
+		})
+	})
+
+	diags = diags.Extend(diag.FromErr(err))
+	return diags
+}
+
+func rewriteComputeIdToClusterId(v dyn.Value, p dyn.Path) (dyn.Value, diag.Diagnostics) {
+	var diags diag.Diagnostics
+	computeIdPath := p.Append(dyn.Key("compute_id"))
+	computeId, err := dyn.GetByPath(v, computeIdPath)
+
+	// If the "compute_id" key is not set, we don't need to do anything.
+	if err != nil {
+		return v, nil
+	}
+
+	if computeId.Kind() == dyn.KindInvalid {
+		return v, nil
+	}
+
+	diags = diags.Append(diag.Diagnostic{
+		Severity:  diag.Warning,
+		Summary:   "compute_id is deprecated, please use cluster_id instead",
+		Locations: computeId.Locations(),
+		Paths:     []dyn.Path{computeIdPath},
+	})
+
+	clusterIdPath := p.Append(dyn.Key("cluster_id"))
+	nv, err := dyn.SetByPath(v, clusterIdPath, computeId)
+	if err != nil {
+		return dyn.InvalidValue, diag.FromErr(err)
+	}
+	// Drop the "compute_id" key.
+	vout, err := dyn.Walk(nv, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+		switch len(p) {
+		case 0:
+			return v, nil
+		case 1:
+			if p[0] == dyn.Key("compute_id") {
+				return v, dyn.ErrDrop
+			}
+			return v, nil
+		case 2:
+			if p[1] == dyn.Key("compute_id") {
+				return v, dyn.ErrDrop
+			}
+		}
+		return v, dyn.ErrSkip
+	})
+
+	diags = diags.Extend(diag.FromErr(err))
+	return vout, diags
+}

bundle/config/mutator/compute_id_compate_test.go

@@ -0,0 +1,57 @@
+package mutator_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestComputeIdToClusterId(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Bundle: config.Bundle{
+				ComputeId: "compute-id",
+			},
+		},
+	}
+
+	diags := bundle.Apply(context.Background(), b, mutator.ComputeIdToClusterId())
+	assert.NoError(t, diags.Error())
+	assert.Equal(t, "compute-id", b.Config.Bundle.ClusterId)
+	assert.Empty(t, b.Config.Bundle.ComputeId)
+
+	assert.Len(t, diags, 1)
+	assert.Equal(t, "compute_id is deprecated, please use cluster_id instead", diags[0].Summary)
+	assert.Equal(t, diag.Warning, diags[0].Severity)
+}
+
+func TestComputeIdToClusterIdInTargetOverride(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Targets: map[string]*config.Target{
+				"dev": {
+					ComputeId: "compute-id-dev",
+				},
+			},
+		},
+	}
+
+	diags := bundle.Apply(context.Background(), b, mutator.ComputeIdToClusterId())
+	assert.NoError(t, diags.Error())
+	assert.Empty(t, b.Config.Targets["dev"].ComputeId)
+
+	diags = diags.Extend(bundle.Apply(context.Background(), b, mutator.SelectTarget("dev")))
+	assert.NoError(t, diags.Error())
+
+	assert.Equal(t, "compute-id-dev", b.Config.Bundle.ClusterId)
+	assert.Empty(t, b.Config.Bundle.ComputeId)
+
+	assert.Len(t, diags, 1)
+	assert.Equal(t, "compute_id is deprecated, please use cluster_id instead", diags[0].Summary)
+	assert.Equal(t, diag.Warning, diags[0].Severity)
+}

bundle/config/mutator/default_workspace_paths.go

@@ -29,6 +29,10 @@ func (m *defineDefaultWorkspacePaths) Apply(ctx context.Context, b *bundle.Bundl
 		b.Config.Workspace.FilePath = path.Join(root, "files")
 	}
 
+	if b.Config.Workspace.ResourcePath == "" {
+		b.Config.Workspace.ResourcePath = path.Join(root, "resources")
+	}
+
 	if b.Config.Workspace.ArtifactPath == "" {
 		b.Config.Workspace.ArtifactPath = path.Join(root, "artifacts")
 	}

bundle/config/mutator/default_workspace_paths_test.go

@@ -22,6 +22,7 @@ func TestDefineDefaultWorkspacePaths(t *testing.T) {
 	diags := bundle.Apply(context.Background(), b, mutator.DefineDefaultWorkspacePaths())
 	require.NoError(t, diags.Error())
 	assert.Equal(t, "/files", b.Config.Workspace.FilePath)
+	assert.Equal(t, "/resources", b.Config.Workspace.ResourcePath)
 	assert.Equal(t, "/artifacts", b.Config.Workspace.ArtifactPath)
 	assert.Equal(t, "/state", b.Config.Workspace.StatePath)
 }
@@ -32,6 +33,7 @@ func TestDefineDefaultWorkspacePathsAlreadySet(t *testing.T) {
 			Workspace: config.Workspace{
 				RootPath:     "/",
 				FilePath:     "/foo/bar",
+				ResourcePath: "/foo/bar",
 				ArtifactPath: "/foo/bar",
 				StatePath:    "/foo/bar",
 			},
@@ -40,6 +42,7 @@ func TestDefineDefaultWorkspacePathsAlreadySet(t *testing.T) {
 	diags := bundle.Apply(context.Background(), b, mutator.DefineDefaultWorkspacePaths())
 	require.NoError(t, diags.Error())
 	assert.Equal(t, "/foo/bar", b.Config.Workspace.FilePath)
+	assert.Equal(t, "/foo/bar", b.Config.Workspace.ResourcePath)
 	assert.Equal(t, "/foo/bar", b.Config.Workspace.ArtifactPath)
 	assert.Equal(t, "/foo/bar", b.Config.Workspace.StatePath)
 }

bundle/config/mutator/expand_pipeline_glob_paths_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/resources"
 	"github.com/databricks/cli/bundle/internal/bundletest"
+	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/pipelines"
 	"github.com/stretchr/testify/require"
@@ -41,7 +42,7 @@ func TestExpandGlobPathsInPipelines(t *testing.T) {
 	touchEmptyFile(t, filepath.Join(dir, "skip/test7.py"))
 
 	b := &bundle.Bundle{
-		RootPath: dir,
+		BundleRootPath: dir,
 		Config: config.Root{
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
@@ -105,8 +106,8 @@ func TestExpandGlobPathsInPipelines(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml"))
+	bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}})
-	bundletest.SetLocation(b, "resources.pipelines.pipeline.libraries[3]", filepath.Join(dir, "relative", "resource.yml"))
+	bundletest.SetLocation(b, "resources.pipelines.pipeline.libraries[3]", []dyn.Location{{File: filepath.Join(dir, "relative", "resource.yml")}})
 
 	m := ExpandPipelineGlobPaths()
 	diags := bundle.Apply(context.Background(), b, m)

bundle/config/mutator/expand_workspace_root.go

@@ -33,7 +33,7 @@ func (m *expandWorkspaceRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.
 	}
 
 	if strings.HasPrefix(root, "~/") {
-		home := fmt.Sprintf("/Users/%s", currentUser.UserName)
+		home := fmt.Sprintf("/Workspace/Users/%s", currentUser.UserName)
 		b.Config.Workspace.RootPath = path.Join(home, root[2:])
 	}
 

bundle/config/mutator/expand_workspace_root_test.go

@@ -27,7 +27,7 @@ func TestExpandWorkspaceRoot(t *testing.T) {
 	}
 	diags := bundle.Apply(context.Background(), b, mutator.ExpandWorkspaceRoot())
 	require.NoError(t, diags.Error())
-	assert.Equal(t, "/Users/jane@doe.com/foo", b.Config.Workspace.RootPath)
+	assert.Equal(t, "/Workspace/Users/jane@doe.com/foo", b.Config.Workspace.RootPath)
 }
 
 func TestExpandWorkspaceRootDoesNothing(t *testing.T) {

bundle/config/mutator/load_git_details.go

@@ -56,7 +56,7 @@ func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn
 	}
 
 	// Compute relative path of the bundle root from the Git repo root.
-	absBundlePath, err := filepath.Abs(b.RootPath)
+	absBundlePath, err := filepath.Abs(b.BundleRootPath)
 	if err != nil {
 		return diag.FromErr(err)
 	}

bundle/config/mutator/mutator.go

@@ -23,6 +23,7 @@ func DefaultMutators() []bundle.Mutator {
 		VerifyCliVersion(),
 
 		EnvironmentsToTargets(),
+		ComputeIdToClusterId(),
 		InitializeVariables(),
 		DefineDefaultTarget(),
 		LoadGitDetails(),

bundle/config/mutator/override_compute.go

@@ -39,22 +39,22 @@ func overrideJobCompute(j *resources.Job, compute string) {
 
 func (m *overrideCompute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	if b.Config.Bundle.Mode != config.Development {
-		if b.Config.Bundle.ComputeID != "" {
+		if b.Config.Bundle.ClusterId != "" {
 			return diag.Errorf("cannot override compute for an target that does not use 'mode: development'")
 		}
 		return nil
 	}
 	if v := env.Get(ctx, "DATABRICKS_CLUSTER_ID"); v != "" {
-		b.Config.Bundle.ComputeID = v
+		b.Config.Bundle.ClusterId = v
 	}
 
-	if b.Config.Bundle.ComputeID == "" {
+	if b.Config.Bundle.ClusterId == "" {
 		return nil
 	}
 
 	r := b.Config.Resources
 	for i := range r.Jobs {
-		overrideJobCompute(r.Jobs[i], b.Config.Bundle.ComputeID)
+		overrideJobCompute(r.Jobs[i], b.Config.Bundle.ClusterId)
 	}
 
 	return nil

bundle/config/mutator/override_compute_test.go

@@ -20,7 +20,7 @@ func TestOverrideDevelopment(t *testing.T) {
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Mode:      config.Development,
-				ComputeID: "newClusterID",
+				ClusterId: "newClusterID",
 			},
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
@@ -144,7 +144,7 @@ func TestOverrideProduction(t *testing.T) {
 	b := &bundle.Bundle{
 		Config: config.Root{
 			Bundle: config.Bundle{
-				ComputeID: "newClusterID",
+				ClusterId: "newClusterID",
 			},
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{

bundle/config/mutator/paths/job_paths_visitor.go

@@ -0,0 +1,115 @@
+package paths
+
+import (
+	"github.com/databricks/cli/bundle/libraries"
+	"github.com/databricks/cli/libs/dyn"
+)
+
+type jobRewritePattern struct {
+	pattern     dyn.Pattern
+	kind        PathKind
+	skipRewrite func(string) bool
+}
+
+func noSkipRewrite(string) bool {
+	return false
+}
+
+func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern {
+	return []jobRewritePattern{
+		{
+			base.Append(dyn.Key("notebook_task"), dyn.Key("notebook_path")),
+			PathKindNotebook,
+			noSkipRewrite,
+		},
+		{
+			base.Append(dyn.Key("spark_python_task"), dyn.Key("python_file")),
+			PathKindWorkspaceFile,
+			noSkipRewrite,
+		},
+		{
+			base.Append(dyn.Key("dbt_task"), dyn.Key("project_directory")),
+			PathKindDirectory,
+			noSkipRewrite,
+		},
+		{
+			base.Append(dyn.Key("sql_task"), dyn.Key("file"), dyn.Key("path")),
+			PathKindWorkspaceFile,
+			noSkipRewrite,
+		},
+		{
+			base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("whl")),
+			PathKindLibrary,
+			noSkipRewrite,
+		},
+		{
+			base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("jar")),
+			PathKindLibrary,
+			noSkipRewrite,
+		},
+		{
+			base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")),
+			PathKindWorkspaceFile,
+			noSkipRewrite,
+		},
+	}
+}
+
+func jobRewritePatterns() []jobRewritePattern {
+	// Base pattern to match all tasks in all jobs.
+	base := dyn.NewPattern(
+		dyn.Key("resources"),
+		dyn.Key("jobs"),
+		dyn.AnyKey(),
+		dyn.Key("tasks"),
+		dyn.AnyIndex(),
+	)
+
+	// Compile list of patterns and their respective rewrite functions.
+	jobEnvironmentsPatterns := []jobRewritePattern{
+		{
+			dyn.NewPattern(
+				dyn.Key("resources"),
+				dyn.Key("jobs"),
+				dyn.AnyKey(),
+				dyn.Key("environments"),
+				dyn.AnyIndex(),
+				dyn.Key("spec"),
+				dyn.Key("dependencies"),
+				dyn.AnyIndex(),
+			),
+			PathKindWithPrefix,
+			func(s string) bool {
+				return !libraries.IsLibraryLocal(s)
+			},
+		},
+	}
+
+	taskPatterns := jobTaskRewritePatterns(base)
+	forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task")))
+	allPatterns := append(taskPatterns, jobEnvironmentsPatterns...)
+	allPatterns = append(allPatterns, forEachPatterns...)
+	return allPatterns
+}
+
+// VisitJobPaths visits all paths in job resources and applies a function to each path.
+func VisitJobPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) {
+	var err error
+	var newValue = value
+
+	for _, rewritePattern := range jobRewritePatterns() {
+		newValue, err = dyn.MapByPattern(newValue, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+			if rewritePattern.skipRewrite(v.MustString()) {
+				return v, nil
+			}
+
+			return fn(p, rewritePattern.kind, v)
+		})
+
+		if err != nil {
+			return dyn.InvalidValue, err
+		}
+	}
+
+	return newValue, nil
+}

bundle/config/mutator/paths/job_paths_visitor_test.go

@@ -0,0 +1,168 @@
+package paths
+
+import (
+	"testing"
+
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/libs/dyn"
+	assert "github.com/databricks/cli/libs/dyn/dynassert"
+	"github.com/databricks/databricks-sdk-go/service/compute"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/stretchr/testify/require"
+)
+
+func TestVisitJobPaths(t *testing.T) {
+	task0 := jobs.Task{
+		NotebookTask: &jobs.NotebookTask{
+			NotebookPath: "abc",
+		},
+	}
+	task1 := jobs.Task{
+		SparkPythonTask: &jobs.SparkPythonTask{
+			PythonFile: "abc",
+		},
+	}
+	task2 := jobs.Task{
+		DbtTask: &jobs.DbtTask{
+			ProjectDirectory: "abc",
+		},
+	}
+	task3 := jobs.Task{
+		SqlTask: &jobs.SqlTask{
+			File: &jobs.SqlTaskFile{
+				Path: "abc",
+			},
+		},
+	}
+	task4 := jobs.Task{
+		Libraries: []compute.Library{
+			{Whl: "dist/foo.whl"},
+		},
+	}
+	task5 := jobs.Task{
+		Libraries: []compute.Library{
+			{Jar: "dist/foo.jar"},
+		},
+	}
+	task6 := jobs.Task{
+		Libraries: []compute.Library{
+			{Requirements: "requirements.txt"},
+		},
+	}
+
+	job0 := &resources.Job{
+		JobSettings: &jobs.JobSettings{
+			Tasks: []jobs.Task{
+				task0,
+				task1,
+				task2,
+				task3,
+				task4,
+				task5,
+				task6,
+			},
+		},
+	}
+
+	root := config.Root{
+		Resources: config.Resources{
+			Jobs: map[string]*resources.Job{
+				"job0": job0,
+			},
+		},
+	}
+
+	actual := visitJobPaths(t, root)
+	expected := []dyn.Path{
+		dyn.MustPathFromString("resources.jobs.job0.tasks[0].notebook_task.notebook_path"),
+		dyn.MustPathFromString("resources.jobs.job0.tasks[1].spark_python_task.python_file"),
+		dyn.MustPathFromString("resources.jobs.job0.tasks[2].dbt_task.project_directory"),
+		dyn.MustPathFromString("resources.jobs.job0.tasks[3].sql_task.file.path"),
+		dyn.MustPathFromString("resources.jobs.job0.tasks[4].libraries[0].whl"),
+		dyn.MustPathFromString("resources.jobs.job0.tasks[5].libraries[0].jar"),
+		dyn.MustPathFromString("resources.jobs.job0.tasks[6].libraries[0].requirements"),
+	}
+
+	assert.ElementsMatch(t, expected, actual)
+}
+
+func TestVisitJobPaths_environments(t *testing.T) {
+	environment0 := jobs.JobEnvironment{
+		Spec: &compute.Environment{
+			Dependencies: []string{
+				"dist_0/*.whl",
+				"dist_1/*.whl",
+			},
+		},
+	}
+	job0 := &resources.Job{
+		JobSettings: &jobs.JobSettings{
+			Environments: []jobs.JobEnvironment{
+				environment0,
+			},
+		},
+	}
+
+	root := config.Root{
+		Resources: config.Resources{
+			Jobs: map[string]*resources.Job{
+				"job0": job0,
+			},
+		},
+	}
+
+	actual := visitJobPaths(t, root)
+	expected := []dyn.Path{
+		dyn.MustPathFromString("resources.jobs.job0.environments[0].spec.dependencies[0]"),
+		dyn.MustPathFromString("resources.jobs.job0.environments[0].spec.dependencies[1]"),
+	}
+
+	assert.ElementsMatch(t, expected, actual)
+}
+
+func TestVisitJobPaths_foreach(t *testing.T) {
+	task0 := jobs.Task{
+		ForEachTask: &jobs.ForEachTask{
+			Task: jobs.Task{
+				NotebookTask: &jobs.NotebookTask{
+					NotebookPath: "abc",
+				},
+			},
+		},
+	}
+	job0 := &resources.Job{
+		JobSettings: &jobs.JobSettings{
+			Tasks: []jobs.Task{
+				task0,
+			},
+		},
+	}
+
+	root := config.Root{
+		Resources: config.Resources{
+			Jobs: map[string]*resources.Job{
+				"job0": job0,
+			},
+		},
+	}
+
+	actual := visitJobPaths(t, root)
+	expected := []dyn.Path{
+		dyn.MustPathFromString("resources.jobs.job0.tasks[0].for_each_task.task.notebook_task.notebook_path"),
+	}
+
+	assert.ElementsMatch(t, expected, actual)
+}
+
+func visitJobPaths(t *testing.T, root config.Root) []dyn.Path {
+	var actual []dyn.Path
+	err := root.Mutate(func(value dyn.Value) (dyn.Value, error) {
+		return VisitJobPaths(value, func(p dyn.Path, kind PathKind, v dyn.Value) (dyn.Value, error) {
+			actual = append(actual, p)
+			return v, nil
+		})
+	})
+	require.NoError(t, err)
+	return actual
+}

bundle/config/mutator/paths/visitor.go

@@ -0,0 +1,26 @@
+package paths
+
+import "github.com/databricks/cli/libs/dyn"
+
+type PathKind int
+
+const (
+	// PathKindLibrary is a path to a library file
+	PathKindLibrary = iota
+
+	// PathKindNotebook is a path to a notebook file
+	PathKindNotebook
+
+	// PathKindWorkspaceFile is a path to a regular workspace file,
+	// notebooks are not allowed because they are uploaded a special
+	// kind of workspace object.
+	PathKindWorkspaceFile
+
+	// PathKindWithPrefix is a path that starts with './'
+	PathKindWithPrefix
+
+	// PathKindDirectory is a path to directory
+	PathKindDirectory
+)
+
+type VisitFunc func(path dyn.Path, kind PathKind, value dyn.Value) (dyn.Value, error)

bundle/config/mutator/populate_current_user.go

@@ -5,8 +5,8 @@ import (
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
-	"github.com/databricks/cli/libs/auth"
 	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/iamutil"
 	"github.com/databricks/cli/libs/tags"
 )
 
@@ -33,7 +33,7 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag.
 	}
 
 	b.Config.Workspace.CurrentUser = &config.User{
-		ShortName: auth.GetShortUserName(me.UserName),
+		ShortName: iamutil.GetShortUserName(me),
 		User:      me,
 	}
 

bundle/config/mutator/prepend_workspace_prefix.go

@@ -0,0 +1,67 @@
+package mutator
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+)
+
+type prependWorkspacePrefix struct{}
+
+// PrependWorkspacePrefix prepends the workspace root path to all paths in the bundle.
+func PrependWorkspacePrefix() bundle.Mutator {
+	return &prependWorkspacePrefix{}
+}
+
+func (m *prependWorkspacePrefix) Name() string {
+	return "PrependWorkspacePrefix"
+}
+
+var skipPrefixes = []string{
+	"/Workspace/",
+	"/Volumes/",
+}
+
+func (m *prependWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	patterns := []dyn.Pattern{
+		dyn.NewPattern(dyn.Key("workspace"), dyn.Key("root_path")),
+		dyn.NewPattern(dyn.Key("workspace"), dyn.Key("file_path")),
+		dyn.NewPattern(dyn.Key("workspace"), dyn.Key("artifact_path")),
+		dyn.NewPattern(dyn.Key("workspace"), dyn.Key("state_path")),
+	}
+
+	err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
+		var err error
+		for _, pattern := range patterns {
+			v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, pv dyn.Value) (dyn.Value, error) {
+				path, ok := pv.AsString()
+				if !ok {
+					return dyn.InvalidValue, fmt.Errorf("expected string, got %s", v.Kind())
+				}
+
+				for _, prefix := range skipPrefixes {
+					if strings.HasPrefix(path, prefix) {
+						return pv, nil
+					}
+				}
+
+				return dyn.NewValue(fmt.Sprintf("/Workspace%s", path), v.Locations()), nil
+			})
+
+			if err != nil {
+				return dyn.InvalidValue, err
+			}
+		}
+		return v, nil
+	})
+
+	if err != nil {
+		return diag.FromErr(err)
+	}
+
+	return nil
+}

bundle/config/mutator/prepend_workspace_prefix_test.go

@@ -0,0 +1,79 @@
+package mutator
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/databricks-sdk-go/service/iam"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPrependWorkspacePrefix(t *testing.T) {
+	testCases := []struct {
+		path     string
+		expected string
+	}{
+		{
+			path:     "/Users/test",
+			expected: "/Workspace/Users/test",
+		},
+		{
+			path:     "/Shared/test",
+			expected: "/Workspace/Shared/test",
+		},
+		{
+			path:     "/Workspace/Users/test",
+			expected: "/Workspace/Users/test",
+		},
+		{
+			path:     "/Volumes/Users/test",
+			expected: "/Volumes/Users/test",
+		},
+	}
+
+	for _, tc := range testCases {
+		b := &bundle.Bundle{
+			Config: config.Root{
+				Workspace: config.Workspace{
+					RootPath:     tc.path,
+					ArtifactPath: tc.path,
+					FilePath:     tc.path,
+					StatePath:    tc.path,
+				},
+			},
+		}
+
+		diags := bundle.Apply(context.Background(), b, PrependWorkspacePrefix())
+		require.Empty(t, diags)
+		require.Equal(t, tc.expected, b.Config.Workspace.RootPath)
+		require.Equal(t, tc.expected, b.Config.Workspace.ArtifactPath)
+		require.Equal(t, tc.expected, b.Config.Workspace.FilePath)
+		require.Equal(t, tc.expected, b.Config.Workspace.StatePath)
+	}
+}
+
+func TestPrependWorkspaceForDefaultConfig(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Bundle: config.Bundle{
+				Name:   "test",
+				Target: "dev",
+			},
+			Workspace: config.Workspace{
+				CurrentUser: &config.User{
+					User: &iam.User{
+						UserName: "jane@doe.com",
+					},
+				},
+			},
+		},
+	}
+	diags := bundle.Apply(context.Background(), b, bundle.Seq(DefineDefaultWorkspaceRoot(), ExpandWorkspaceRoot(), DefineDefaultWorkspacePaths(), PrependWorkspacePrefix()))
+	require.Empty(t, diags)
+	require.Equal(t, "/Workspace/Users/jane@doe.com/.bundle/test/dev", b.Config.Workspace.RootPath)
+	require.Equal(t, "/Workspace/Users/jane@doe.com/.bundle/test/dev/artifacts", b.Config.Workspace.ArtifactPath)
+	require.Equal(t, "/Workspace/Users/jane@doe.com/.bundle/test/dev/files", b.Config.Workspace.FilePath)
+	require.Equal(t, "/Workspace/Users/jane@doe.com/.bundle/test/dev/state", b.Config.Workspace.StatePath)
+}

bundle/config/mutator/process_target_mode.go

@@ -6,9 +6,9 @@ import (
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
-	"github.com/databricks/cli/libs/auth"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/iamutil"
 	"github.com/databricks/cli/libs/log"
 )
 
@@ -118,15 +118,18 @@ func findNonUserPath(b *bundle.Bundle) string {
 	if b.Config.Workspace.RootPath != "" && !containsName(b.Config.Workspace.RootPath) {
 		return "root_path"
 	}
-	if b.Config.Workspace.StatePath != "" && !containsName(b.Config.Workspace.StatePath) {
-		return "state_path"
-	}
 	if b.Config.Workspace.FilePath != "" && !containsName(b.Config.Workspace.FilePath) {
 		return "file_path"
 	}
+	if b.Config.Workspace.ResourcePath != "" && !containsName(b.Config.Workspace.ResourcePath) {
+		return "resource_path"
+	}
 	if b.Config.Workspace.ArtifactPath != "" && !containsName(b.Config.Workspace.ArtifactPath) {
 		return "artifact_path"
 	}
+	if b.Config.Workspace.StatePath != "" && !containsName(b.Config.Workspace.StatePath) {
+		return "state_path"
+	}
 	return ""
 }
 
@@ -171,7 +174,7 @@ func (m *processTargetMode) Apply(ctx context.Context, b *bundle.Bundle) diag.Di
 		transformDevelopmentMode(ctx, b)
 		return diags
 	case config.Production:
-		isPrincipal := auth.IsServicePrincipal(b.Config.Workspace.CurrentUser.UserName)
+		isPrincipal := iamutil.IsServicePrincipal(b.Config.Workspace.CurrentUser.User)
 		return validateProductionMode(ctx, b, isPrincipal)
 	case "":
 		// No action

bundle/config/mutator/process_target_mode_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/databricks/cli/libs/tags"
 	sdkconfig "github.com/databricks/databricks-sdk-go/config"
 	"github.com/databricks/databricks-sdk-go/service/catalog"
+	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/iam"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/databricks/databricks-sdk-go/service/ml"
@@ -122,6 +123,9 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
 				Volumes: map[string]*resources.Volume{
 					"volume1": {CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{Name: "volume1"}},
 				},
+				Clusters: map[string]*resources.Cluster{
+					"cluster1": {ClusterSpec: &compute.ClusterSpec{ClusterName: "cluster1", SparkVersion: "13.2.x", NumWorkers: 1}},
+				},
 			},
 		},
 		// Use AWS implementation for testing.
@@ -180,6 +184,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) {
 
 	// Schema 1
 	assert.Equal(t, "dev_lennart_schema1", b.Config.Resources.Schemas["schema1"].Name)
+
+	// Clusters
+	assert.Equal(t, "[dev lennart] cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName)
 }
 
 func TestProcessTargetModeDevelopmentTagNormalizationForAws(t *testing.T) {
@@ -286,6 +293,7 @@ func TestProcessTargetModeDefault(t *testing.T) {
 	assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName)
 	assert.Equal(t, "schema1", b.Config.Resources.Schemas["schema1"].Name)
 	assert.Equal(t, "volume1", b.Config.Resources.Volumes["volume1"].Name)
+	assert.Equal(t, "cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName)
 }
 
 func TestProcessTargetModeProduction(t *testing.T) {
@@ -317,6 +325,7 @@ func TestProcessTargetModeProduction(t *testing.T) {
 	b.Config.Resources.Experiments["experiment2"].Permissions = permissions
 	b.Config.Resources.Models["model1"].Permissions = permissions
 	b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Permissions = permissions
+	b.Config.Resources.Clusters["cluster1"].Permissions = permissions
 
 	diags = validateProductionMode(context.Background(), b, false)
 	require.NoError(t, diags.Error())
@@ -329,6 +338,7 @@ func TestProcessTargetModeProduction(t *testing.T) {
 	assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName)
 	assert.Equal(t, "schema1", b.Config.Resources.Schemas["schema1"].Name)
 	assert.Equal(t, "volume1", b.Config.Resources.Volumes["volume1"].Name)
+	assert.Equal(t, "cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName)
 }
 
 func TestProcessTargetModeProductionOkForPrincipal(t *testing.T) {

bundle/config/mutator/python/python_mutator.go

@@ -108,7 +108,7 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
 			return dyn.InvalidValue, fmt.Errorf("failed to create cache dir: %w", err)
 		}
 
-		rightRoot, diags := m.runPythonMutator(ctx, cacheDir, b.RootPath, pythonPath, leftRoot)
+		rightRoot, diags := m.runPythonMutator(ctx, cacheDir, b.BundleRootPath, pythonPath, leftRoot)
 		mutateDiags = diags
 		if diags.HasError() {
 			return dyn.InvalidValue, mutateDiagsHasError
@@ -228,12 +228,12 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir string, r
 	return output, pythonDiagnostics
 }
 
-const installExplanation = `If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies, 
+const installExplanation = `If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies,
 and that the wheel is installed in the Python environment:
 
   $ .venv/bin/pip install -e .
 
-If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml, 
+If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml,
 or activate the environment before running CLI commands:
 
   experimental:

bundle/config/mutator/python/python_mutator_test.go

@@ -570,12 +570,12 @@ func TestExplainProcessErr(t *testing.T) {
 
 Explanation: 'databricks-pydabs' library is not installed in the Python environment.
 
-If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies, 
+If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies,
 and that the wheel is installed in the Python environment:
 
   $ .venv/bin/pip install -e .
 
-If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml, 
+If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml,
 or activate the environment before running CLI commands:
 
   experimental:

bundle/config/mutator/rewrite_sync_paths.go

@@ -45,15 +45,15 @@ func (m *rewriteSyncPaths) makeRelativeTo(root string) dyn.MapFunc {
 func (m *rewriteSyncPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
 		return dyn.Map(v, "sync", func(_ dyn.Path, v dyn.Value) (nv dyn.Value, err error) {
-			v, err = dyn.Map(v, "paths", dyn.Foreach(m.makeRelativeTo(b.RootPath)))
+			v, err = dyn.Map(v, "paths", dyn.Foreach(m.makeRelativeTo(b.BundleRootPath)))
 			if err != nil {
 				return dyn.InvalidValue, err
 			}
-			v, err = dyn.Map(v, "include", dyn.Foreach(m.makeRelativeTo(b.RootPath)))
+			v, err = dyn.Map(v, "include", dyn.Foreach(m.makeRelativeTo(b.BundleRootPath)))
 			if err != nil {
 				return dyn.InvalidValue, err
 			}
-			v, err = dyn.Map(v, "exclude", dyn.Foreach(m.makeRelativeTo(b.RootPath)))
+			v, err = dyn.Map(v, "exclude", dyn.Foreach(m.makeRelativeTo(b.BundleRootPath)))
 			if err != nil {
 				return dyn.InvalidValue, err
 			}

bundle/config/mutator/rewrite_sync_paths_test.go

@@ -9,12 +9,13 @@ import (
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/mutator"
 	"github.com/databricks/cli/bundle/internal/bundletest"
+	"github.com/databricks/cli/libs/dyn"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestRewriteSyncPathsRelative(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: ".",
+		BundleRootPath: ".",
 		Config: config.Root{
 			Sync: config.Sync{
 				Paths: []string{
@@ -33,12 +34,12 @@ func TestRewriteSyncPathsRelative(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, "sync.paths[0]", "./databricks.yml")
+	bundletest.SetLocation(b, "sync.paths[0]", []dyn.Location{{File: "./databricks.yml"}})
-	bundletest.SetLocation(b, "sync.paths[1]", "./databricks.yml")
+	bundletest.SetLocation(b, "sync.paths[1]", []dyn.Location{{File: "./databricks.yml"}})
-	bundletest.SetLocation(b, "sync.include[0]", "./file.yml")
+	bundletest.SetLocation(b, "sync.include[0]", []dyn.Location{{File: "./file.yml"}})
-	bundletest.SetLocation(b, "sync.include[1]", "./a/file.yml")
+	bundletest.SetLocation(b, "sync.include[1]", []dyn.Location{{File: "./a/file.yml"}})
-	bundletest.SetLocation(b, "sync.exclude[0]", "./a/b/file.yml")
+	bundletest.SetLocation(b, "sync.exclude[0]", []dyn.Location{{File: "./a/b/file.yml"}})
-	bundletest.SetLocation(b, "sync.exclude[1]", "./a/b/c/file.yml")
+	bundletest.SetLocation(b, "sync.exclude[1]", []dyn.Location{{File: "./a/b/c/file.yml"}})
 
 	diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths())
 	assert.NoError(t, diags.Error())
@@ -53,7 +54,7 @@ func TestRewriteSyncPathsRelative(t *testing.T) {
 
 func TestRewriteSyncPathsAbsolute(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: "/tmp/dir",
+		BundleRootPath: "/tmp/dir",
 		Config: config.Root{
 			Sync: config.Sync{
 				Paths: []string{
@@ -72,12 +73,12 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, "sync.paths[0]", "/tmp/dir/databricks.yml")
+	bundletest.SetLocation(b, "sync.paths[0]", []dyn.Location{{File: "/tmp/dir/databricks.yml"}})
-	bundletest.SetLocation(b, "sync.paths[1]", "/tmp/dir/databricks.yml")
+	bundletest.SetLocation(b, "sync.paths[1]", []dyn.Location{{File: "/tmp/dir/databricks.yml"}})
-	bundletest.SetLocation(b, "sync.include[0]", "/tmp/dir/file.yml")
+	bundletest.SetLocation(b, "sync.include[0]", []dyn.Location{{File: "/tmp/dir/file.yml"}})
-	bundletest.SetLocation(b, "sync.include[1]", "/tmp/dir/a/file.yml")
+	bundletest.SetLocation(b, "sync.include[1]", []dyn.Location{{File: "/tmp/dir/a/file.yml"}})
-	bundletest.SetLocation(b, "sync.exclude[0]", "/tmp/dir/a/b/file.yml")
+	bundletest.SetLocation(b, "sync.exclude[0]", []dyn.Location{{File: "/tmp/dir/a/b/file.yml"}})
-	bundletest.SetLocation(b, "sync.exclude[1]", "/tmp/dir/a/b/c/file.yml")
+	bundletest.SetLocation(b, "sync.exclude[1]", []dyn.Location{{File: "/tmp/dir/a/b/c/file.yml"}})
 
 	diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths())
 	assert.NoError(t, diags.Error())
@@ -93,7 +94,7 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) {
 func TestRewriteSyncPathsErrorPaths(t *testing.T) {
 	t.Run("no sync block", func(t *testing.T) {
 		b := &bundle.Bundle{
-			RootPath: ".",
+			BundleRootPath: ".",
 		}
 
 		diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths())
@@ -102,7 +103,7 @@ func TestRewriteSyncPathsErrorPaths(t *testing.T) {
 
 	t.Run("empty include/exclude blocks", func(t *testing.T) {
 		b := &bundle.Bundle{
-			RootPath: ".",
+			BundleRootPath: ".",
 			Config: config.Root{
 				Sync: config.Sync{
 					Include: []string{},

bundle/config/mutator/rewrite_workspace_prefix.go

@@ -0,0 +1,72 @@
+package mutator
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+)
+
+type rewriteWorkspacePrefix struct{}
+
+// RewriteWorkspacePrefix finds any strings in bundle configration that have
+// workspace prefix plus workspace path variable used and removes workspace prefix from it.
+func RewriteWorkspacePrefix() bundle.Mutator {
+	return &rewriteWorkspacePrefix{}
+}
+
+func (m *rewriteWorkspacePrefix) Name() string {
+	return "RewriteWorkspacePrefix"
+}
+
+func (m *rewriteWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	diags := diag.Diagnostics{}
+	paths := map[string]string{
+		"/Workspace/${workspace.root_path}":     "${workspace.root_path}",
+		"/Workspace${workspace.root_path}":      "${workspace.root_path}",
+		"/Workspace/${workspace.file_path}":     "${workspace.file_path}",
+		"/Workspace${workspace.file_path}":      "${workspace.file_path}",
+		"/Workspace/${workspace.artifact_path}": "${workspace.artifact_path}",
+		"/Workspace${workspace.artifact_path}":  "${workspace.artifact_path}",
+		"/Workspace/${workspace.state_path}":    "${workspace.state_path}",
+		"/Workspace${workspace.state_path}":     "${workspace.state_path}",
+	}
+
+	err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
+		// Walk through the bundle configuration, check all the string leafs and
+		// see if any of the prefixes are used in the remote path.
+		return dyn.Walk(root, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+			vv, ok := v.AsString()
+			if !ok {
+				return v, nil
+			}
+
+			for path, replacePath := range paths {
+				if strings.Contains(vv, path) {
+					newPath := strings.Replace(vv, path, replacePath, 1)
+					diags = append(diags, diag.Diagnostic{
+						Severity:  diag.Warning,
+						Summary:   fmt.Sprintf("substring %q found in %q. Please update this to %q.", path, vv, newPath),
+						Detail:    "For more information, please refer to: https://docs.databricks.com/en/release-notes/dev-tools/bundles.html#workspace-paths",
+						Locations: v.Locations(),
+						Paths:     []dyn.Path{p},
+					})
+
+					// Remove the workspace prefix from the string.
+					return dyn.NewValue(newPath, v.Locations()), nil
+				}
+			}
+
+			return v, nil
+		})
+	})
+
+	if err != nil {
+		return diag.FromErr(err)
+	}
+
+	return diags
+}

bundle/config/mutator/rewrite_workspace_prefix_test.go

@@ -0,0 +1,85 @@
+package mutator
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/databricks-sdk-go/service/compute"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNoWorkspacePrefixUsed(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Workspace: config.Workspace{
+				RootPath:     "/Workspace/Users/test",
+				ArtifactPath: "/Workspace/Users/test/artifacts",
+				FilePath:     "/Workspace/Users/test/files",
+				StatePath:    "/Workspace/Users/test/state",
+			},
+
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"test_job": {
+						JobSettings: &jobs.JobSettings{
+							Tasks: []jobs.Task{
+								{
+									SparkPythonTask: &jobs.SparkPythonTask{
+										PythonFile: "/Workspace/${workspace.root_path}/file1.py",
+									},
+								},
+								{
+									NotebookTask: &jobs.NotebookTask{
+										NotebookPath: "/Workspace${workspace.file_path}/notebook1",
+									},
+									Libraries: []compute.Library{
+										{
+											Jar: "/Workspace/${workspace.artifact_path}/jar1.jar",
+										},
+									},
+								},
+								{
+									NotebookTask: &jobs.NotebookTask{
+										NotebookPath: "${workspace.file_path}/notebook2",
+									},
+									Libraries: []compute.Library{
+										{
+											Jar: "${workspace.artifact_path}/jar2.jar",
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	diags := bundle.Apply(context.Background(), b, RewriteWorkspacePrefix())
+	require.Len(t, diags, 3)
+
+	expectedErrors := map[string]bool{
+		`substring "/Workspace/${workspace.root_path}" found in "/Workspace/${workspace.root_path}/file1.py". Please update this to "${workspace.root_path}/file1.py".`:             true,
+		`substring "/Workspace${workspace.file_path}" found in "/Workspace${workspace.file_path}/notebook1". Please update this to "${workspace.file_path}/notebook1".`:             true,
+		`substring "/Workspace/${workspace.artifact_path}" found in "/Workspace/${workspace.artifact_path}/jar1.jar". Please update this to "${workspace.artifact_path}/jar1.jar".`: true,
+	}
+
+	for _, d := range diags {
+		require.Equal(t, d.Severity, diag.Warning)
+		require.Contains(t, expectedErrors, d.Summary)
+		delete(expectedErrors, d.Summary)
+	}
+
+	require.Equal(t, "${workspace.root_path}/file1.py", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[0].SparkPythonTask.PythonFile)
+	require.Equal(t, "${workspace.file_path}/notebook1", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[1].NotebookTask.NotebookPath)
+	require.Equal(t, "${workspace.artifact_path}/jar1.jar", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[1].Libraries[0].Jar)
+	require.Equal(t, "${workspace.file_path}/notebook2", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[2].NotebookTask.NotebookPath)
+	require.Equal(t, "${workspace.artifact_path}/jar2.jar", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[2].Libraries[0].Jar)
+
+}

bundle/config/mutator/run_as.go

@@ -30,50 +30,44 @@ func (m *setRunAs) Name() string {
 	return "SetRunAs"
 }
 
-type errUnsupportedResourceTypeForRunAs struct {
+func reportRunAsNotSupported(resourceType string, location dyn.Location, currentUser string, runAsUser string) diag.Diagnostics {
-	resourceType     string
+	return diag.Diagnostics{{
-	resourceLocation dyn.Location
+		Summary: fmt.Sprintf("%s do not support a setting a run_as user that is different from the owner.\n"+
-	currentUser      string
+			"Current identity: %s. Run as identity: %s.\n"+
-	runAsUser        string
+			"See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.", resourceType, currentUser, runAsUser),
+		Locations: []dyn.Location{location},
+		Severity:  diag.Error,
+	}}
 }
 
-func (e errUnsupportedResourceTypeForRunAs) Error() string {
+func validateRunAs(b *bundle.Bundle) diag.Diagnostics {
-	return fmt.Sprintf("%s are not supported when the current deployment user is different from the bundle's run_as identity. Please deploy as the run_as identity. Please refer to the documentation at https://docs.databricks.com/dev-tools/bundles/run-as.html for more details. Location of the unsupported resource: %s. Current identity: %s. Run as identity: %s", e.resourceType, e.resourceLocation, e.currentUser, e.runAsUser)
+	diags := diag.Diagnostics{}
-}
 
-type errBothSpAndUserSpecified struct {
+	neitherSpecifiedErr := diag.Diagnostics{{
-	spName   string
+		Summary:   "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified",
-	spLoc    dyn.Location
+		Locations: []dyn.Location{b.Config.GetLocation("run_as")},
-	userName string
+		Severity:  diag.Error,
-	userLoc  dyn.Location
+	}}
-}
 
-func (e errBothSpAndUserSpecified) Error() string {
+	// Fail fast if neither service_principal_name nor user_name are specified, but the
-	return fmt.Sprintf("run_as section must specify exactly one identity. A service_principal_name %q is specified at %s. A user_name %q is defined at %s", e.spName, e.spLoc, e.userName, e.userLoc)
-}
-
-func validateRunAs(b *bundle.Bundle) error {
-	neitherSpecifiedErr := fmt.Errorf("run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s", b.Config.GetLocation("run_as"))
-	// Error if neither service_principal_name nor user_name are specified, but the
 	// run_as section is present.
 	if b.Config.Value().Get("run_as").Kind() == dyn.KindNil {
 		return neitherSpecifiedErr
 	}
-	// Error if one or both of service_principal_name and user_name are specified,
+
+	// Fail fast if one or both of service_principal_name and user_name are specified,
 	// but with empty values.
-	if b.Config.RunAs.ServicePrincipalName == "" && b.Config.RunAs.UserName == "" {
+	runAs := b.Config.RunAs
+	if runAs.ServicePrincipalName == "" && runAs.UserName == "" {
 		return neitherSpecifiedErr
 	}
 
-	// Error if both service_principal_name and user_name are specified
-	runAs := b.Config.RunAs
 	if runAs.UserName != "" && runAs.ServicePrincipalName != "" {
-		return errBothSpAndUserSpecified{
+		diags = diags.Extend(diag.Diagnostics{{
-			spName:   runAs.ServicePrincipalName,
+			Summary:   "run_as section cannot specify both user_name and service_principal_name",
-			userName: runAs.UserName,
+			Locations: []dyn.Location{b.Config.GetLocation("run_as")},
-			spLoc:    b.Config.GetLocation("run_as.service_principal_name"),
+			Severity:  diag.Error,
-			userLoc:  b.Config.GetLocation("run_as.user_name"),
+		}})
-		}
 	}
 
 	identity := runAs.ServicePrincipalName
@@ -83,40 +77,40 @@ func validateRunAs(b *bundle.Bundle) error {
 
 	// All resources are supported if the run_as identity is the same as the current deployment identity.
 	if identity == b.Config.Workspace.CurrentUser.UserName {
-		return nil
+		return diags
 	}
 
 	// DLT pipelines do not support run_as in the API.
 	if len(b.Config.Resources.Pipelines) > 0 {
-		return errUnsupportedResourceTypeForRunAs{
+		diags = diags.Extend(reportRunAsNotSupported(
-			resourceType:     "pipelines",
+			"pipelines",
-			resourceLocation: b.Config.GetLocation("resources.pipelines"),
+			b.Config.GetLocation("resources.pipelines"),
-			currentUser:      b.Config.Workspace.CurrentUser.UserName,
+			b.Config.Workspace.CurrentUser.UserName,
-			runAsUser:        identity,
+			identity,
-		}
+		))
 	}
 
 	// Model serving endpoints do not support run_as in the API.
 	if len(b.Config.Resources.ModelServingEndpoints) > 0 {
-		return errUnsupportedResourceTypeForRunAs{
+		diags = diags.Extend(reportRunAsNotSupported(
-			resourceType:     "model_serving_endpoints",
+			"model_serving_endpoints",
-			resourceLocation: b.Config.GetLocation("resources.model_serving_endpoints"),
+			b.Config.GetLocation("resources.model_serving_endpoints"),
-			currentUser:      b.Config.Workspace.CurrentUser.UserName,
+			b.Config.Workspace.CurrentUser.UserName,
-			runAsUser:        identity,
+			identity,
-		}
+		))
 	}
 
 	// Monitors do not support run_as in the API.
 	if len(b.Config.Resources.QualityMonitors) > 0 {
-		return errUnsupportedResourceTypeForRunAs{
+		diags = diags.Extend(reportRunAsNotSupported(
-			resourceType:     "quality_monitors",
+			"quality_monitors",
-			resourceLocation: b.Config.GetLocation("resources.quality_monitors"),
+			b.Config.GetLocation("resources.quality_monitors"),
-			currentUser:      b.Config.Workspace.CurrentUser.UserName,
+			b.Config.Workspace.CurrentUser.UserName,
-			runAsUser:        identity,
+			identity,
-		}
+		))
 	}
 
-	return nil
+	return diags
 }
 
 func setRunAsForJobs(b *bundle.Bundle) {
@@ -187,8 +181,9 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
 	}
 
 	// Assert the run_as configuration is valid in the context of the bundle
-	if err := validateRunAs(b); err != nil {
+	diags := validateRunAs(b)
-		return diag.FromErr(err)
+	if diags.HasError() {
+		return diags
 	}
 
 	setRunAsForJobs(b)

bundle/config/mutator/run_as_test.go

@@ -32,6 +32,7 @@ func allResourceTypes(t *testing.T) []string {
 	// the dyn library gives us the correct list of all resources supported. Please
 	// also update this check when adding a new resource
 	require.Equal(t, []string{
+		"clusters",
 		"experiments",
 		"jobs",
 		"model_serving_endpoints",
@@ -134,6 +135,7 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) {
 	// some point in the future. These resources are (implicitly) on the deny list, since
 	// they are not on the allow list below.
 	allowList := []string{
+		"clusters",
 		"jobs",
 		"models",
 		"registered_models",
@@ -188,11 +190,8 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) {
 			Config: *r,
 		}
 		diags := bundle.Apply(context.Background(), b, SetRunAs())
-		assert.Equal(t, diags.Error().Error(), errUnsupportedResourceTypeForRunAs{
+		assert.Contains(t, diags.Error().Error(), "do not support a setting a run_as user that is different from the owner.\n"+
-			resourceType:     rt,
+			"Current identity: alice. Run as identity: bob.\n"+
-			resourceLocation: dyn.Location{},
+			"See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.", rt)
-			currentUser:      "alice",
-			runAsUser:        "bob",
-		}.Error(), "expected run_as with a different identity than the current deployment user to not supported for resources of type: %s", rt)
 	}
 }

bundle/config/mutator/sync_default_path_test.go

@@ -15,8 +15,8 @@ import (
 
 func TestSyncDefaultPath_DefaultIfUnset(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: "/tmp/some/dir",
+		BundleRootPath: "/tmp/some/dir",
-		Config:   config.Root{},
+		Config:         config.Root{},
 	}
 
 	ctx := context.Background()
@@ -51,8 +51,8 @@ func TestSyncDefaultPath_SkipIfSet(t *testing.T) {
 	for _, tcase := range tcases {
 		t.Run(tcase.name, func(t *testing.T) {
 			b := &bundle.Bundle{
-				RootPath: "/tmp/some/dir",
+				BundleRootPath: "/tmp/some/dir",
-				Config:   config.Root{},
+				Config:         config.Root{},
 			}
 
 			diags := bundle.ApplyFunc(context.Background(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {

bundle/config/mutator/sync_infer_root.go

@@ -57,7 +57,7 @@ func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
 	var diags diag.Diagnostics
 
 	// Use the bundle root path as the starting point for inferring the sync root path.
-	bundleRootPath := filepath.Clean(b.RootPath)
+	bundleRootPath := filepath.Clean(b.BundleRootPath)
 
 	// Infer the sync root path by looking at each one of the sync paths.
 	// Every sync path must be a descendant of the final sync root path.

bundle/config/mutator/sync_infer_root_test.go

@@ -9,13 +9,14 @@ import (
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/mutator"
 	"github.com/databricks/cli/bundle/internal/bundletest"
+	"github.com/databricks/cli/libs/dyn"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestSyncInferRoot_NominalAbsolute(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: "/tmp/some/dir",
+		BundleRootPath: "/tmp/some/dir",
 		Config: config.Root{
 			Sync: config.Sync{
 				Paths: []string{
@@ -46,7 +47,7 @@ func TestSyncInferRoot_NominalAbsolute(t *testing.T) {
 
 func TestSyncInferRoot_NominalRelative(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: "./some/dir",
+		BundleRootPath: "./some/dir",
 		Config: config.Root{
 			Sync: config.Sync{
 				Paths: []string{
@@ -77,7 +78,7 @@ func TestSyncInferRoot_NominalRelative(t *testing.T) {
 
 func TestSyncInferRoot_ParentDirectory(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: "/tmp/some/dir",
+		BundleRootPath: "/tmp/some/dir",
 		Config: config.Root{
 			Sync: config.Sync{
 				Paths: []string{
@@ -108,7 +109,7 @@ func TestSyncInferRoot_ParentDirectory(t *testing.T) {
 
 func TestSyncInferRoot_ManyParentDirectories(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: "/tmp/some/dir/that/is/very/deeply/nested",
+		BundleRootPath: "/tmp/some/dir/that/is/very/deeply/nested",
 		Config: config.Root{
 			Sync: config.Sync{
 				Paths: []string{
@@ -145,7 +146,7 @@ func TestSyncInferRoot_ManyParentDirectories(t *testing.T) {
 
 func TestSyncInferRoot_MultiplePaths(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: "/tmp/some/bundle/root",
+		BundleRootPath: "/tmp/some/bundle/root",
 		Config: config.Root{
 			Sync: config.Sync{
 				Paths: []string{
@@ -172,7 +173,7 @@ func TestSyncInferRoot_MultiplePaths(t *testing.T) {
 
 func TestSyncInferRoot_Error(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: "/tmp/some/dir",
+		BundleRootPath: "/tmp/some/dir",
 		Config: config.Root{
 			Sync: config.Sync{
 				Paths: []string{
@@ -184,7 +185,7 @@ func TestSyncInferRoot_Error(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, "sync.paths", "databricks.yml")
+	bundletest.SetLocation(b, "sync.paths", []dyn.Location{{File: "databricks.yml"}})
 
 	ctx := context.Background()
 	diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())

bundle/config/mutator/translate_paths_jobs.go

@@ -4,97 +4,11 @@ import (
 	"fmt"
 	"slices"
 
-	"github.com/databricks/cli/bundle/libraries"
+	"github.com/databricks/cli/bundle/config/mutator/paths"
+
 	"github.com/databricks/cli/libs/dyn"
 )
 
-type jobRewritePattern struct {
-	pattern     dyn.Pattern
-	fn          rewriteFunc
-	skipRewrite func(string) bool
-}
-
-func noSkipRewrite(string) bool {
-	return false
-}
-
-func rewritePatterns(t *translateContext, base dyn.Pattern) []jobRewritePattern {
-	return []jobRewritePattern{
-		{
-			base.Append(dyn.Key("notebook_task"), dyn.Key("notebook_path")),
-			t.translateNotebookPath,
-			noSkipRewrite,
-		},
-		{
-			base.Append(dyn.Key("spark_python_task"), dyn.Key("python_file")),
-			t.translateFilePath,
-			noSkipRewrite,
-		},
-		{
-			base.Append(dyn.Key("dbt_task"), dyn.Key("project_directory")),
-			t.translateDirectoryPath,
-			noSkipRewrite,
-		},
-		{
-			base.Append(dyn.Key("sql_task"), dyn.Key("file"), dyn.Key("path")),
-			t.translateFilePath,
-			noSkipRewrite,
-		},
-		{
-			base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("whl")),
-			t.translateNoOp,
-			noSkipRewrite,
-		},
-		{
-			base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("jar")),
-			t.translateNoOp,
-			noSkipRewrite,
-		},
-		{
-			base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")),
-			t.translateFilePath,
-			noSkipRewrite,
-		},
-	}
-}
-
-func (t *translateContext) jobRewritePatterns() []jobRewritePattern {
-	// Base pattern to match all tasks in all jobs.
-	base := dyn.NewPattern(
-		dyn.Key("resources"),
-		dyn.Key("jobs"),
-		dyn.AnyKey(),
-		dyn.Key("tasks"),
-		dyn.AnyIndex(),
-	)
-
-	// Compile list of patterns and their respective rewrite functions.
-	jobEnvironmentsPatterns := []jobRewritePattern{
-		{
-			dyn.NewPattern(
-				dyn.Key("resources"),
-				dyn.Key("jobs"),
-				dyn.AnyKey(),
-				dyn.Key("environments"),
-				dyn.AnyIndex(),
-				dyn.Key("spec"),
-				dyn.Key("dependencies"),
-				dyn.AnyIndex(),
-			),
-			t.translateNoOpWithPrefix,
-			func(s string) bool {
-				return !libraries.IsLibraryLocal(s)
-			},
-		},
-	}
-
-	taskPatterns := rewritePatterns(t, base)
-	forEachPatterns := rewritePatterns(t, base.Append(dyn.Key("for_each_task"), dyn.Key("task")))
-	allPatterns := append(taskPatterns, jobEnvironmentsPatterns...)
-	allPatterns = append(allPatterns, forEachPatterns...)
-	return allPatterns
-}
-
 func (t *translateContext) applyJobTranslations(v dyn.Value) (dyn.Value, error) {
 	var err error
 
@@ -111,30 +25,41 @@ func (t *translateContext) applyJobTranslations(v dyn.Value) (dyn.Value, error)
 		}
 	}
 
-	for _, rewritePattern := range t.jobRewritePatterns() {
+	return paths.VisitJobPaths(v, func(p dyn.Path, kind paths.PathKind, v dyn.Value) (dyn.Value, error) {
-		v, err = dyn.MapByPattern(v, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+		key := p[2].Key()
-			key := p[2].Key()
 
-			// Skip path translation if the job is using git source.
+		// Skip path translation if the job is using git source.
-			if slices.Contains(ignore, key) {
+		if slices.Contains(ignore, key) {
-				return v, nil
+			return v, nil
-			}
+		}
 
-			dir, err := v.Location().Directory()
+		dir, err := v.Location().Directory()
-			if err != nil {
+		if err != nil {
-				return dyn.InvalidValue, fmt.Errorf("unable to determine directory for job %s: %w", key, err)
+			return dyn.InvalidValue, fmt.Errorf("unable to determine directory for job %s: %w", key, err)
-			}
+		}
 
-			sv := v.MustString()
+		rewritePatternFn, err := t.getRewritePatternFn(kind)
-			if rewritePattern.skipRewrite(sv) {
-				return v, nil
-			}
-			return t.rewriteRelativeTo(p, v, rewritePattern.fn, dir, fallback[key])
-		})
 		if err != nil {
 			return dyn.InvalidValue, err
 		}
+
+		return t.rewriteRelativeTo(p, v, rewritePatternFn, dir, fallback[key])
+	})
+}
+
+func (t *translateContext) getRewritePatternFn(kind paths.PathKind) (rewriteFunc, error) {
+	switch kind {
+	case paths.PathKindLibrary:
+		return t.translateNoOp, nil
+	case paths.PathKindNotebook:
+		return t.translateNotebookPath, nil
+	case paths.PathKindWorkspaceFile:
+		return t.translateFilePath, nil
+	case paths.PathKindDirectory:
+		return t.translateDirectoryPath, nil
+	case paths.PathKindWithPrefix:
+		return t.translateNoOpWithPrefix, nil
 	}
 
-	return v, nil
+	return nil, fmt.Errorf("unsupported path kind: %d", kind)
 }

bundle/config/mutator/translate_paths_test.go

@@ -82,7 +82,7 @@ func TestTranslatePathsSkippedWithGitSource(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml"))
+	bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}})
 
 	diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
 	require.NoError(t, diags.Error())
@@ -210,7 +210,7 @@ func TestTranslatePaths(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml"))
+	bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}})
 
 	diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
 	require.NoError(t, diags.Error())
@@ -346,8 +346,8 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, "resources.jobs", filepath.Join(dir, "job/resource.yml"))
+	bundletest.SetLocation(b, "resources.jobs", []dyn.Location{{File: filepath.Join(dir, "job/resource.yml")}})
-	bundletest.SetLocation(b, "resources.pipelines", filepath.Join(dir, "pipeline/resource.yml"))
+	bundletest.SetLocation(b, "resources.pipelines", []dyn.Location{{File: filepath.Join(dir, "pipeline/resource.yml")}})
 
 	diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
 	require.NoError(t, diags.Error())
@@ -408,7 +408,7 @@ func TestTranslatePathsOutsideSyncRoot(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, ".", filepath.Join(dir, "../resource.yml"))
+	bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "../resource.yml")}})
 
 	diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
 	assert.ErrorContains(t, diags.Error(), "is not contained in sync root path")
@@ -439,7 +439,7 @@ func TestJobNotebookDoesNotExistError(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, ".", filepath.Join(dir, "fake.yml"))
+	bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}})
 
 	diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
 	assert.EqualError(t, diags.Error(), "notebook ./doesnt_exist.py not found")
@@ -470,7 +470,7 @@ func TestJobFileDoesNotExistError(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, ".", filepath.Join(dir, "fake.yml"))
+	bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}})
 
 	diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
 	assert.EqualError(t, diags.Error(), "file ./doesnt_exist.py not found")
@@ -501,7 +501,7 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, ".", filepath.Join(dir, "fake.yml"))
+	bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}})
 
 	diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
 	assert.EqualError(t, diags.Error(), "notebook ./doesnt_exist.py not found")
@@ -532,7 +532,7 @@ func TestPipelineFileDoesNotExistError(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, ".", filepath.Join(dir, "fake.yml"))
+	bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}})
 
 	diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
 	assert.EqualError(t, diags.Error(), "file ./doesnt_exist.py not found")
@@ -567,7 +567,7 @@ func TestJobSparkPythonTaskWithNotebookSourceError(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml"))
+	bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}})
 
 	diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
 	assert.ErrorContains(t, diags.Error(), `expected a file for "resources.jobs.job.tasks[0].spark_python_task.python_file" but got a notebook`)
@@ -602,7 +602,7 @@ func TestJobNotebookTaskWithFileSourceError(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml"))
+	bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}})
 
 	diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
 	assert.ErrorContains(t, diags.Error(), `expected a notebook for "resources.jobs.job.tasks[0].notebook_task.notebook_path" but got a file`)
@@ -637,7 +637,7 @@ func TestPipelineNotebookLibraryWithFileSourceError(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml"))
+	bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}})
 
 	diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
 	assert.ErrorContains(t, diags.Error(), `expected a notebook for "resources.pipelines.pipeline.libraries[0].notebook.path" but got a file`)
@@ -672,7 +672,7 @@ func TestPipelineFileLibraryWithNotebookSourceError(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml"))
+	bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}})
 
 	diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
 	assert.ErrorContains(t, diags.Error(), `expected a file for "resources.pipelines.pipeline.libraries[0].file.path" but got a notebook`)
@@ -710,7 +710,7 @@ func TestTranslatePathJobEnvironments(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, "resources.jobs", filepath.Join(dir, "job/resource.yml"))
+	bundletest.SetLocation(b, "resources.jobs", []dyn.Location{{File: filepath.Join(dir, "job/resource.yml")}})
 
 	diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
 	require.NoError(t, diags.Error())
@@ -753,8 +753,8 @@ func TestTranslatePathWithComplexVariables(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, "variables", filepath.Join(dir, "variables/variables.yml"))
+	bundletest.SetLocation(b, "variables", []dyn.Location{{File: filepath.Join(dir, "variables/variables.yml")}})
-	bundletest.SetLocation(b, "resources.jobs", filepath.Join(dir, "job/resource.yml"))
+	bundletest.SetLocation(b, "resources.jobs", []dyn.Location{{File: filepath.Join(dir, "job/resource.yml")}})
 
 	ctx := context.Background()
 	// Assign the variables to the dynamic configuration.

bundle/config/resources.go

@@ -20,6 +20,7 @@ type Resources struct {
 	QualityMonitors       map[string]*resources.QualityMonitor       `json:"quality_monitors,omitempty"`
 	Schemas               map[string]*resources.Schema               `json:"schemas,omitempty"`
 	Volumes               map[string]*resources.Volume               `json:"volumes,omitempty"`
+	Clusters              map[string]*resources.Cluster              `json:"clusters,omitempty"`
 }
 
 type ConfigResource interface {
@@ -59,3 +60,22 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error)
 
 	return found[0], nil
 }
+
+type ResourceDescription struct {
+	SingularName string
+}
+
+// The keys of the map corresponds to the resource key in the bundle configuration.
+func SupportedResources() map[string]ResourceDescription {
+	return map[string]ResourceDescription{
+		"jobs":                    {SingularName: "job"},
+		"pipelines":               {SingularName: "pipeline"},
+		"models":                  {SingularName: "model"},
+		"experiments":             {SingularName: "experiment"},
+		"model_serving_endpoints": {SingularName: "model_serving_endpoint"},
+		"registered_models":       {SingularName: "registered_model"},
+		"quality_monitors":        {SingularName: "quality_monitor"},
+		"schemas":                 {SingularName: "schema"},
+		"clusters":                {SingularName: "cluster"},
+	}
+}

bundle/config/resources/clusters.go

@@ -0,0 +1,39 @@
+package resources
+
+import (
+	"context"
+
+	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/marshal"
+	"github.com/databricks/databricks-sdk-go/service/compute"
+)
+
+type Cluster struct {
+	ID             string         `json:"id,omitempty" bundle:"readonly"`
+	Permissions    []Permission   `json:"permissions,omitempty"`
+	ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
+
+	*compute.ClusterSpec
+}
+
+func (s *Cluster) UnmarshalJSON(b []byte) error {
+	return marshal.Unmarshal(b, s)
+}
+
+func (s Cluster) MarshalJSON() ([]byte, error) {
+	return marshal.Marshal(s)
+}
+
+func (s *Cluster) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
+	_, err := w.Clusters.GetByClusterId(ctx, id)
+	if err != nil {
+		log.Debugf(ctx, "cluster %s does not exist", id)
+		return false, err
+	}
+	return true, nil
+}
+
+func (s *Cluster) TerraformResourceName() string {
+	return "databricks_cluster"
+}

bundle/config/resources_test.go

@@ -3,6 +3,7 @@ package config
 import (
 	"encoding/json"
 	"reflect"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -61,3 +62,18 @@ func TestCustomMarshallerIsImplemented(t *testing.T) {
 		}, "Resource %s does not have a custom unmarshaller", field.Name)
 	}
 }
+
+func TestSupportedResources(t *testing.T) {
+	expected := map[string]ResourceDescription{}
+	typ := reflect.TypeOf(Resources{})
+	for i := 0; i < typ.NumField(); i++ {
+		field := typ.Field(i)
+		jsonTags := strings.Split(field.Tag.Get("json"), ",")
+		singularName := strings.TrimSuffix(jsonTags[0], "s")
+		expected[jsonTags[0]] = ResourceDescription{SingularName: singularName}
+	}
+
+	// Please add your resource to the SupportedResources() function in resources.go
+	// if you are adding a new resource.
+	assert.Equal(t, expected, SupportedResources())
+}

bundle/config/root.go

@@ -366,9 +366,9 @@ func (r *Root) MergeTargetOverrides(name string) error {
 		}
 	}
 
-	// Merge `compute_id`. This field must be overwritten if set, not merged.
+	// Merge `cluster_id`. This field must be overwritten if set, not merged.
-	if v := target.Get("compute_id"); v.Kind() != dyn.KindInvalid {
+	if v := target.Get("cluster_id"); v.Kind() != dyn.KindInvalid {
-		root, err = dyn.SetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("compute_id")), v)
+		root, err = dyn.SetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("cluster_id")), v)
 		if err != nil {
 			return err
 		}
@@ -406,23 +406,45 @@ func (r *Root) MergeTargetOverrides(name string) error {
 	return r.updateWithDynamicValue(root)
 }
 
-var variableKeywords = []string{"default", "lookup"}
+var allowedVariableDefinitions = []([]string){
+	{"default", "type", "description"},
+	{"default", "type"},
+	{"default", "description"},
+	{"lookup", "description"},
+	{"default"},
+	{"lookup"},
+}
 
 // isFullVariableOverrideDef checks if the given value is a full syntax varaible override.
-// A full syntax variable override is a map with only one of the following
+// A full syntax variable override is a map with either 1 of 2 keys.
-// keys: "default", "lookup".
+// If it's 2 keys, the keys should be "default" and "type".
+// If it's 1 key, the key should be one of the following keys: "default", "lookup".
 func isFullVariableOverrideDef(v dyn.Value) bool {
 	mv, ok := v.AsMap()
 	if !ok {
 		return false
 	}
 
-	if mv.Len() != 1 {
+	// If the map has more than 3 keys, it is not a full variable override.
+	if mv.Len() > 3 {
 		return false
 	}
 
-	for _, keyword := range variableKeywords {
+	for _, keys := range allowedVariableDefinitions {
-		if _, ok := mv.GetByString(keyword); ok {
+		if len(keys) != mv.Len() {
+			continue
+		}
+
+		// Check if the keys are the same.
+		match := true
+		for _, key := range keys {
+			if _, ok := mv.GetByString(key); !ok {
+				match = false
+				break
+			}
+		}
+
+		if match {
 			return true
 		}
 	}

bundle/config/root_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/databricks/cli/bundle/config/variable"
+	"github.com/databricks/cli/libs/dyn"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -169,3 +170,87 @@ func TestRootMergeTargetOverridesWithVariables(t *testing.T) {
 	assert.Equal(t, "complex var", root.Variables["complex"].Description)
 
 }
+
+func TestIsFullVariableOverrideDef(t *testing.T) {
+	testCases := []struct {
+		value    dyn.Value
+		expected bool
+	}{
+		{
+			value: dyn.V(map[string]dyn.Value{
+				"type":        dyn.V("string"),
+				"default":     dyn.V("foo"),
+				"description": dyn.V("foo var"),
+			}),
+			expected: true,
+		},
+		{
+			value: dyn.V(map[string]dyn.Value{
+				"type":        dyn.V("string"),
+				"lookup":      dyn.V("foo"),
+				"description": dyn.V("foo var"),
+			}),
+			expected: false,
+		},
+		{
+			value: dyn.V(map[string]dyn.Value{
+				"type":    dyn.V("string"),
+				"default": dyn.V("foo"),
+			}),
+			expected: true,
+		},
+		{
+			value: dyn.V(map[string]dyn.Value{
+				"type":   dyn.V("string"),
+				"lookup": dyn.V("foo"),
+			}),
+			expected: false,
+		},
+		{
+			value: dyn.V(map[string]dyn.Value{
+				"description": dyn.V("string"),
+				"default":     dyn.V("foo"),
+			}),
+			expected: true,
+		},
+		{
+			value: dyn.V(map[string]dyn.Value{
+				"description": dyn.V("string"),
+				"lookup":      dyn.V("foo"),
+			}),
+			expected: true,
+		},
+		{
+			value: dyn.V(map[string]dyn.Value{
+				"default": dyn.V("foo"),
+			}),
+			expected: true,
+		},
+		{
+			value: dyn.V(map[string]dyn.Value{
+				"lookup": dyn.V("foo"),
+			}),
+			expected: true,
+		},
+		{
+			value: dyn.V(map[string]dyn.Value{
+				"type": dyn.V("string"),
+			}),
+			expected: false,
+		},
+		{
+			value: dyn.V(map[string]dyn.Value{
+				"type":        dyn.V("string"),
+				"default":     dyn.V("foo"),
+				"description": dyn.V("foo var"),
+				"lookup":      dyn.V("foo"),
+			}),
+			expected: false,
+		},
+	}
+
+	for i, tc := range testCases {
+		assert.Equal(t, tc.expected, isFullVariableOverrideDef(tc.value), "test case %d", i)
+	}
+
+}

bundle/config/target.go

@@ -24,8 +24,11 @@ type Target struct {
 	// name prefix of deployed resources.
 	Presets Presets `json:"presets,omitempty"`
 
-	// Overrides the compute used for jobs and other supported assets.
+	// DEPRECATED: Overrides the compute used for jobs and other supported assets.
-	ComputeID string `json:"compute_id,omitempty"`
+	ComputeId string `json:"compute_id,omitempty"`
+
+	// Overrides the cluster used for jobs and other supported assets.
+	ClusterId string `json:"cluster_id,omitempty"`
 
 	Bundle *Bundle `json:"bundle,omitempty"`
 

bundle/config/validate/job_task_cluster_spec.go

@@ -0,0 +1,161 @@
+package validate
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+)
+
+// JobTaskClusterSpec validates that job tasks have cluster spec defined
+// if task requires a cluster
+func JobTaskClusterSpec() bundle.ReadOnlyMutator {
+	return &jobTaskClusterSpec{}
+}
+
+type jobTaskClusterSpec struct {
+}
+
+func (v *jobTaskClusterSpec) Name() string {
+	return "validate:job_task_cluster_spec"
+}
+
+func (v *jobTaskClusterSpec) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics {
+	diags := diag.Diagnostics{}
+
+	jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"))
+
+	for resourceName, job := range rb.Config().Resources.Jobs {
+		resourcePath := jobsPath.Append(dyn.Key(resourceName))
+
+		for taskIndex, task := range job.Tasks {
+			taskPath := resourcePath.Append(dyn.Key("tasks"), dyn.Index(taskIndex))
+
+			diags = diags.Extend(validateJobTask(rb, task, taskPath))
+		}
+	}
+
+	return diags
+}
+
+func validateJobTask(rb bundle.ReadOnlyBundle, task jobs.Task, taskPath dyn.Path) diag.Diagnostics {
+	diags := diag.Diagnostics{}
+
+	var specified []string
+	var unspecified []string
+
+	if task.JobClusterKey != "" {
+		specified = append(specified, "job_cluster_key")
+	} else {
+		unspecified = append(unspecified, "job_cluster_key")
+	}
+
+	if task.EnvironmentKey != "" {
+		specified = append(specified, "environment_key")
+	} else {
+		unspecified = append(unspecified, "environment_key")
+	}
+
+	if task.ExistingClusterId != "" {
+		specified = append(specified, "existing_cluster_id")
+	} else {
+		unspecified = append(unspecified, "existing_cluster_id")
+	}
+
+	if task.NewCluster != nil {
+		specified = append(specified, "new_cluster")
+	} else {
+		unspecified = append(unspecified, "new_cluster")
+	}
+
+	if task.ForEachTask != nil {
+		forEachTaskPath := taskPath.Append(dyn.Key("for_each_task"), dyn.Key("task"))
+
+		diags = diags.Extend(validateJobTask(rb, task.ForEachTask.Task, forEachTaskPath))
+	}
+
+	if isComputeTask(task) && len(specified) == 0 {
+		if task.NotebookTask != nil {
+			// notebook tasks without cluster spec will use notebook environment
+		} else {
+			// path might be not very helpful, adding user-specified task key clarifies the context
+			detail := fmt.Sprintf(
+				"Task %q requires a cluster or an environment to run.\nSpecify one of the following fields: %s.",
+				task.TaskKey,
+				strings.Join(unspecified, ", "),
+			)
+
+			diags = diags.Append(diag.Diagnostic{
+				Severity:  diag.Error,
+				Summary:   "Missing required cluster or environment settings",
+				Detail:    detail,
+				Locations: rb.Config().GetLocations(taskPath.String()),
+				Paths:     []dyn.Path{taskPath},
+			})
+		}
+	}
+
+	return diags
+}
+
+// isComputeTask returns true if the task runs on a cluster or serverless GC
+func isComputeTask(task jobs.Task) bool {
+	if task.NotebookTask != nil {
+		// if warehouse_id is set, it's SQL notebook that doesn't need cluster or serverless GC
+		if task.NotebookTask.WarehouseId != "" {
+			return false
+		} else {
+			// task settings don't require specifying a cluster/serverless GC, but task itself can run on one
+			// we handle that case separately in validateJobTask
+			return true
+		}
+	}
+
+	if task.PythonWheelTask != nil {
+		return true
+	}
+
+	if task.DbtTask != nil {
+		return true
+	}
+
+	if task.SparkJarTask != nil {
+		return true
+	}
+
+	if task.SparkSubmitTask != nil {
+		return true
+	}
+
+	if task.SparkPythonTask != nil {
+		return true
+	}
+
+	if task.SqlTask != nil {
+		return false
+	}
+
+	if task.PipelineTask != nil {
+		// while pipelines use clusters, pipeline tasks don't, they only trigger pipelines
+		return false
+	}
+
+	if task.RunJobTask != nil {
+		return false
+	}
+
+	if task.ConditionTask != nil {
+		return false
+	}
+
+	// for each task doesn't use clusters, underlying task(s) can though
+	if task.ForEachTask != nil {
+		return false
+	}
+
+	return false
+}

bundle/config/validate/job_task_cluster_spec_test.go

@@ -0,0 +1,203 @@
+package validate
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/databricks-sdk-go/service/compute"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestJobTaskClusterSpec(t *testing.T) {
+	expectedSummary := "Missing required cluster or environment settings"
+
+	type testCase struct {
+		name         string
+		task         jobs.Task
+		errorPath    string
+		errorDetail  string
+		errorSummary string
+	}
+
+	testCases := []testCase{
+		{
+			name: "valid notebook task",
+			task: jobs.Task{
+				// while a cluster is needed, it will use notebook environment to create one
+				NotebookTask: &jobs.NotebookTask{},
+			},
+		},
+		{
+			name: "valid notebook task (job_cluster_key)",
+			task: jobs.Task{
+				JobClusterKey: "cluster1",
+				NotebookTask:  &jobs.NotebookTask{},
+			},
+		},
+		{
+			name: "valid notebook task (new_cluster)",
+			task: jobs.Task{
+				NewCluster:   &compute.ClusterSpec{},
+				NotebookTask: &jobs.NotebookTask{},
+			},
+		},
+		{
+			name: "valid notebook task (existing_cluster_id)",
+			task: jobs.Task{
+				ExistingClusterId: "cluster1",
+				NotebookTask:      &jobs.NotebookTask{},
+			},
+		},
+		{
+			name: "valid SQL notebook task",
+			task: jobs.Task{
+				NotebookTask: &jobs.NotebookTask{
+					WarehouseId: "warehouse1",
+				},
+			},
+		},
+		{
+			name: "valid python wheel task",
+			task: jobs.Task{
+				JobClusterKey:   "cluster1",
+				PythonWheelTask: &jobs.PythonWheelTask{},
+			},
+		},
+		{
+			name: "valid python wheel task (environment_key)",
+			task: jobs.Task{
+				EnvironmentKey:  "environment1",
+				PythonWheelTask: &jobs.PythonWheelTask{},
+			},
+		},
+		{
+			name: "valid dbt task",
+			task: jobs.Task{
+				JobClusterKey: "cluster1",
+				DbtTask:       &jobs.DbtTask{},
+			},
+		},
+		{
+			name: "valid spark jar task",
+			task: jobs.Task{
+				JobClusterKey: "cluster1",
+				SparkJarTask:  &jobs.SparkJarTask{},
+			},
+		},
+		{
+			name: "valid spark submit",
+			task: jobs.Task{
+				NewCluster:      &compute.ClusterSpec{},
+				SparkSubmitTask: &jobs.SparkSubmitTask{},
+			},
+		},
+		{
+			name: "valid spark python task",
+			task: jobs.Task{
+				JobClusterKey:   "cluster1",
+				SparkPythonTask: &jobs.SparkPythonTask{},
+			},
+		},
+		{
+			name: "valid SQL task",
+			task: jobs.Task{
+				SqlTask: &jobs.SqlTask{},
+			},
+		},
+		{
+			name: "valid pipeline task",
+			task: jobs.Task{
+				PipelineTask: &jobs.PipelineTask{},
+			},
+		},
+		{
+			name: "valid run job task",
+			task: jobs.Task{
+				RunJobTask: &jobs.RunJobTask{},
+			},
+		},
+		{
+			name: "valid condition task",
+			task: jobs.Task{
+				ConditionTask: &jobs.ConditionTask{},
+			},
+		},
+		{
+			name: "valid for each task",
+			task: jobs.Task{
+				ForEachTask: &jobs.ForEachTask{
+					Task: jobs.Task{
+						JobClusterKey: "cluster1",
+						NotebookTask:  &jobs.NotebookTask{},
+					},
+				},
+			},
+		},
+		{
+			name: "invalid python wheel task",
+			task: jobs.Task{
+				PythonWheelTask: &jobs.PythonWheelTask{},
+				TaskKey:         "my_task",
+			},
+			errorPath: "resources.jobs.job1.tasks[0]",
+			errorDetail: `Task "my_task" requires a cluster or an environment to run.
+Specify one of the following fields: job_cluster_key, environment_key, existing_cluster_id, new_cluster.`,
+			errorSummary: expectedSummary,
+		},
+		{
+			name: "invalid for each task",
+			task: jobs.Task{
+				ForEachTask: &jobs.ForEachTask{
+					Task: jobs.Task{
+						PythonWheelTask: &jobs.PythonWheelTask{},
+						TaskKey:         "my_task",
+					},
+				},
+			},
+			errorPath: "resources.jobs.job1.tasks[0].for_each_task.task",
+			errorDetail: `Task "my_task" requires a cluster or an environment to run.
+Specify one of the following fields: job_cluster_key, environment_key, existing_cluster_id, new_cluster.`,
+			errorSummary: expectedSummary,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			job := &resources.Job{
+				JobSettings: &jobs.JobSettings{
+					Tasks: []jobs.Task{tc.task},
+				},
+			}
+
+			b := createBundle(map[string]*resources.Job{"job1": job})
+			diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobTaskClusterSpec())
+
+			if tc.errorPath != "" || tc.errorDetail != "" || tc.errorSummary != "" {
+				assert.Len(t, diags, 1)
+				assert.Len(t, diags[0].Paths, 1)
+
+				diag := diags[0]
+
+				assert.Equal(t, tc.errorPath, diag.Paths[0].String())
+				assert.Equal(t, tc.errorSummary, diag.Summary)
+				assert.Equal(t, tc.errorDetail, diag.Detail)
+			} else {
+				assert.ElementsMatch(t, []string{}, diags)
+			}
+		})
+	}
+}
+
+func createBundle(jobs map[string]*resources.Job) *bundle.Bundle {
+	return &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Jobs: jobs,
+			},
+		},
+	}
+}

bundle/config/validate/validate.go

@@ -34,6 +34,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 		JobClusterKeyDefined(),
 		FilesToSync(),
 		ValidateSyncPatterns(),
+		JobTaskClusterSpec(),
 	))
 }
 

bundle/config/workspace.go

@@ -47,13 +47,18 @@ type Workspace struct {
 
 	// Remote workspace base path for deployment state, for artifacts, as synchronization target.
 	// This defaults to "~/.bundle/${bundle.name}/${bundle.target}" where "~" expands to
-	// the current user's home directory in the workspace (e.g. `/Users/jane@doe.com`).
+	// the current user's home directory in the workspace (e.g. `/Workspace/Users/jane@doe.com`).
 	RootPath string `json:"root_path,omitempty"`
 
 	// Remote workspace path to synchronize local files to.
 	// This defaults to "${workspace.root}/files".
 	FilePath string `json:"file_path,omitempty"`
 
+	// Remote workspace path for resources with a presence in the workspace.
+	// These are kept outside [FilePath] to avoid potential naming collisions.
+	// This defaults to "${workspace.root}/resources".
+	ResourcePath string `json:"resource_path,omitempty"`
+
 	// Remote workspace path for build artifacts.
 	// This defaults to "${workspace.root}/artifacts".
 	ArtifactPath string `json:"artifact_path,omitempty"`

bundle/deploy/files/upload.go

@@ -2,15 +2,21 @@ package files
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"io/fs"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/permissions"
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/cli/libs/sync"
 )
 
-type upload struct{}
+type upload struct {
+	outputHandler sync.OutputHandler
+}
 
 func (m *upload) Name() string {
 	return "files.Upload"
@@ -18,13 +24,23 @@ func (m *upload) Name() string {
 
 func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	cmdio.LogString(ctx, fmt.Sprintf("Uploading bundle files to %s...", b.Config.Workspace.FilePath))
-	sync, err := GetSync(ctx, bundle.ReadOnly(b))
+	opts, err := GetSyncOptions(ctx, bundle.ReadOnly(b))
 	if err != nil {
 		return diag.FromErr(err)
 	}
 
+	opts.OutputHandler = m.outputHandler
+	sync, err := sync.New(ctx, *opts)
+	if err != nil {
+		return diag.FromErr(err)
+	}
+	defer sync.Close()
+
 	b.Files, err = sync.RunOnce(ctx)
 	if err != nil {
+		if errors.Is(err, fs.ErrPermission) {
+			return permissions.ReportPossiblePermissionDenied(ctx, b, b.Config.Workspace.FilePath)
+		}
 		return diag.FromErr(err)
 	}
 
@@ -32,6 +48,6 @@ func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	return nil
 }
 
-func Upload() bundle.Mutator {
+func Upload(outputHandler sync.OutputHandler) bundle.Mutator {
-	return &upload{}
+	return &upload{outputHandler}
 }

bundle/deploy/lock/acquire.go

@@ -3,8 +3,10 @@ package lock
 import (
 	"context"
 	"errors"
+	"io/fs"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/permissions"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/filer"
 	"github.com/databricks/cli/libs/locker"
@@ -51,12 +53,17 @@ func (m *acquire) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 	if err != nil {
 		log.Errorf(ctx, "Failed to acquire deployment lock: %v", err)
 
+		if errors.Is(err, fs.ErrPermission) {
+			return permissions.ReportPossiblePermissionDenied(ctx, b, b.Config.Workspace.StatePath)
+		}
+
 		notExistsError := filer.NoSuchDirectoryError{}
 		if errors.As(err, &notExistsError) {
 			// If we get a "doesn't exist" error from the API this indicates
 			// we either don't have permissions or the path is invalid.
-			return diag.Errorf("cannot write to deployment root (this can indicate a previous deploy was done with a different identity): %s", b.Config.Workspace.RootPath)
+			return permissions.ReportPossiblePermissionDenied(ctx, b, b.Config.Workspace.StatePath)
 		}
+
 		return diag.FromErr(err)
 	}
 

bundle/deploy/metadata/compute.go

@@ -40,7 +40,7 @@ func (m *compute) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
 		// Compute config file path the job is defined in, relative to the bundle
 		// root
 		l := b.Config.GetLocation("resources.jobs." + name)
-		relativePath, err := filepath.Rel(b.RootPath, l.File)
+		relativePath, err := filepath.Rel(b.BundleRootPath, l.File)
 		if err != nil {
 			return diag.Errorf("failed to compute relative path for job %s: %v", name, err)
 		}

bundle/deploy/metadata/compute_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/databricks/cli/bundle/config/resources"
 	"github.com/databricks/cli/bundle/internal/bundletest"
 	"github.com/databricks/cli/bundle/metadata"
+	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -55,9 +56,9 @@ func TestComputeMetadataMutator(t *testing.T) {
 		},
 	}
 
-	bundletest.SetLocation(b, "resources.jobs.my-job-1", "a/b/c")
+	bundletest.SetLocation(b, "resources.jobs.my-job-1", []dyn.Location{{File: "a/b/c"}})
-	bundletest.SetLocation(b, "resources.jobs.my-job-2", "d/e/f")
+	bundletest.SetLocation(b, "resources.jobs.my-job-2", []dyn.Location{{File: "d/e/f"}})
-	bundletest.SetLocation(b, "resources.pipelines.my-pipeline", "abc")
+	bundletest.SetLocation(b, "resources.pipelines.my-pipeline", []dyn.Location{{File: "abc"}})
 
 	expectedMetadata := metadata.Metadata{
 		Version: metadata.Version,

bundle/deploy/state_pull_test.go

@@ -62,8 +62,8 @@ func testStatePull(t *testing.T, opts statePullOpts) {
 
 	tmpDir := t.TempDir()
 	b := &bundle.Bundle{
-		RootPath:   tmpDir,
+		BundleRootPath: tmpDir,
-		BundleRoot: vfs.MustNew(tmpDir),
+		BundleRoot:     vfs.MustNew(tmpDir),
 
 		SyncRootPath: tmpDir,
 		SyncRoot:     vfs.MustNew(tmpDir),
@@ -259,7 +259,7 @@ func TestStatePullNoState(t *testing.T) {
 	}}
 
 	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target: "default",
@@ -447,7 +447,7 @@ func TestStatePullNewerDeploymentStateVersion(t *testing.T) {
 	}}
 
 	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target: "default",

bundle/deploy/state_push.go

@@ -10,6 +10,8 @@ import (
 	"github.com/databricks/cli/libs/log"
 )
 
+const MaxStateFileSize = 10 * 1024 * 1024 // 10MB
+
 type statePush struct {
 	filerFactory FilerFactory
 }
@@ -35,6 +37,17 @@ func (s *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic
 	}
 	defer local.Close()
 
+	if !b.Config.Bundle.Force {
+		state, err := local.Stat()
+		if err != nil {
+			return diag.FromErr(err)
+		}
+
+		if state.Size() > MaxStateFileSize {
+			return diag.Errorf("Deployment state file size exceeds the maximum allowed size of %d bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag.", MaxStateFileSize)
+		}
+	}
+
 	log.Infof(ctx, "Writing local deployment state file to remote state directory")
 	err = f.Write(ctx, DeploymentStateFileName, local, filer.CreateParentDirectories, filer.OverwriteIfExists)
 	if err != nil {

bundle/deploy/state_push_test.go

@@ -45,7 +45,7 @@ func TestStatePush(t *testing.T) {
 	}}
 
 	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target: "default",

bundle/deploy/state_update_test.go

@@ -27,7 +27,7 @@ func setupBundleForStateUpdate(t *testing.T) *bundle.Bundle {
 	require.NoError(t, err)
 
 	return &bundle.Bundle{
-		RootPath: tmpDir,
+		BundleRootPath: tmpDir,
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target: "default",

bundle/deploy/terraform/apply.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/permissions"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/log"
 	"github.com/hashicorp/terraform-exec/tfexec"
@@ -34,6 +35,10 @@ func (w *apply) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	// Apply terraform according to the computed plan
 	err := tf.Apply(ctx, tfexec.DirOrPlan(b.Plan.Path))
 	if err != nil {
+		diags := permissions.TryExtendTerraformPermissionError(ctx, b, err)
+		if diags != nil {
+			return diags
+		}
 		return diag.Errorf("terraform apply: %v", err)
 	}
 

bundle/deploy/terraform/convert.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"sort"
 
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/resources"
@@ -82,6 +83,10 @@ func BundleToTerraform(config *config.Root) *schema.Root {
 		conv(src, &dst)
 
 		if src.JobSettings != nil {
+			sort.Slice(src.JobSettings.Tasks, func(i, j int) bool {
+				return src.JobSettings.Tasks[i].TaskKey < src.JobSettings.Tasks[j].TaskKey
+			})
+
 			for _, v := range src.Tasks {
 				var t schema.ResourceJobTask
 				conv(v, &t)
@@ -231,6 +236,13 @@ func BundleToTerraform(config *config.Root) *schema.Root {
 		tfroot.Resource.QualityMonitor[k] = &dst
 	}
 
+	for k, src := range config.Resources.Clusters {
+		noResources = false
+		var dst schema.ResourceCluster
+		conv(src, &dst)
+		tfroot.Resource.Cluster[k] = &dst
+	}
+
 	// We explicitly set "resource" to nil to omit it from a JSON encoding.
 	// This is required because the terraform CLI requires >= 1 resources defined
 	// if the "resource" property is used in a .tf.json file.
@@ -404,6 +416,16 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error {
 				}
 				cur.ID = instance.Attributes.ID
 				config.Resources.Volumes[resource.Name] = cur
+			case "databricks_cluster":
+				if config.Resources.Clusters == nil {
+					config.Resources.Clusters = make(map[string]*resources.Cluster)
+				}
+				cur := config.Resources.Clusters[resource.Name]
+				if cur == nil {
+					cur = &resources.Cluster{ModifiedStatus: resources.ModifiedStatusDeleted}
+				}
+				cur.ID = instance.Attributes.ID
+				config.Resources.Clusters[resource.Name] = cur
 			case "databricks_permissions":
 			case "databricks_grants":
 				// Ignore; no need to pull these back into the configuration.
@@ -458,6 +480,11 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error {
 			src.ModifiedStatus = resources.ModifiedStatusCreated
 		}
 	}
+	for _, src := range config.Resources.Clusters {
+		if src.ModifiedStatus == "" && src.ID == "" {
+			src.ModifiedStatus = resources.ModifiedStatusCreated
+		}
+	}
 
 	return nil
 }

bundle/deploy/terraform/convert_test.go

@@ -671,6 +671,14 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) {
 					{Attributes: stateInstanceAttributes{ID: "1"}},
 				},
 			},
+			{
+				Type: "databricks_cluster",
+				Mode: "managed",
+				Name: "test_cluster",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "1"}},
+				},
+			},
 		},
 	}
 	err := TerraformToBundle(&tfState, &config)
@@ -703,6 +711,9 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) {
 	assert.Equal(t, "1", config.Resources.Volumes["test_volume"].ID)
 	assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Volumes["test_volume"].ModifiedStatus)
 
+	assert.Equal(t, "1", config.Resources.Clusters["test_cluster"].ID)
+	assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Clusters["test_cluster"].ModifiedStatus)
+
 	AssertFullResourceCoverage(t, &config)
 }
 
@@ -772,6 +783,13 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
 					},
 				},
 			},
+			Clusters: map[string]*resources.Cluster{
+				"test_cluster": {
+					ClusterSpec: &compute.ClusterSpec{
+						ClusterName: "test_cluster",
+					},
+				},
+			},
 		},
 	}
 	var tfState = resourcesState{
@@ -807,6 +825,9 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
 	assert.Equal(t, "", config.Resources.Volumes["test_volume"].ID)
 	assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Volumes["test_volume"].ModifiedStatus)
 
+	assert.Equal(t, "", config.Resources.Clusters["test_cluster"].ID)
+	assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Clusters["test_cluster"].ModifiedStatus)
+
 	AssertFullResourceCoverage(t, &config)
 }
 
@@ -921,6 +942,18 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
 					},
 				},
 			},
+			Clusters: map[string]*resources.Cluster{
+				"test_cluster": {
+					ClusterSpec: &compute.ClusterSpec{
+						ClusterName: "test_cluster",
+					},
+				},
+				"test_cluster_new": {
+					ClusterSpec: &compute.ClusterSpec{
+						ClusterName: "test_cluster_new",
+					},
+				},
+			},
 		},
 	}
 	var tfState = resourcesState{
@@ -1061,6 +1094,14 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
 					{Attributes: stateInstanceAttributes{ID: "1"}},
 				},
 			},
+			{
+				Type: "databricks_cluster",
+				Mode: "managed",
+				Name: "test_cluster",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "1"}},
+				},
+			},
 			{
 				Type: "databricks_volume",
 				Mode: "managed",
@@ -1069,6 +1110,13 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
 					{Attributes: stateInstanceAttributes{ID: "2"}},
 				},
 			},
+			{Type: "databricks_cluster",
+				Mode: "managed",
+				Name: "test_cluster_old",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "2"}},
+				},
+			},
 		},
 	}
 	err := TerraformToBundle(&tfState, &config)
@@ -1137,6 +1185,13 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
 	assert.Equal(t, "", config.Resources.Volumes["test_volume_new"].ID)
 	assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Volumes["test_volume_new"].ModifiedStatus)
 
+	assert.Equal(t, "1", config.Resources.Clusters["test_cluster"].ID)
+	assert.Equal(t, "", config.Resources.Clusters["test_cluster"].ModifiedStatus)
+	assert.Equal(t, "2", config.Resources.Clusters["test_cluster_old"].ID)
+	assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Clusters["test_cluster_old"].ModifiedStatus)
+	assert.Equal(t, "", config.Resources.Clusters["test_cluster_new"].ID)
+	assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Clusters["test_cluster_new"].ModifiedStatus)
+
 	AssertFullResourceCoverage(t, &config)
 }
 

bundle/deploy/terraform/init_test.go

@@ -33,7 +33,7 @@ func TestInitEnvironmentVariables(t *testing.T) {
 	}
 
 	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target: "whatever",
@@ -60,7 +60,7 @@ func TestSetTempDirEnvVarsForUnixWithTmpDirSet(t *testing.T) {
 	}
 
 	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target: "whatever",
@@ -88,7 +88,7 @@ func TestSetTempDirEnvVarsForUnixWithTmpDirNotSet(t *testing.T) {
 	}
 
 	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target: "whatever",
@@ -114,7 +114,7 @@ func TestSetTempDirEnvVarsForWindowWithAllTmpDirEnvVarsSet(t *testing.T) {
 	}
 
 	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target: "whatever",
@@ -144,7 +144,7 @@ func TestSetTempDirEnvVarsForWindowWithUserProfileAndTempSet(t *testing.T) {
 	}
 
 	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target: "whatever",
@@ -174,7 +174,7 @@ func TestSetTempDirEnvVarsForWindowsWithoutAnyTempDirEnvVarsSet(t *testing.T) {
 	}
 
 	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target: "whatever",
@@ -202,7 +202,7 @@ func TestSetTempDirEnvVarsForWindowsWithoutAnyTempDirEnvVarsSet(t *testing.T) {
 
 func TestSetProxyEnvVars(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target: "whatever",
@@ -250,7 +250,7 @@ func TestSetProxyEnvVars(t *testing.T) {
 
 func TestSetUserAgentExtraEnvVar(t *testing.T) {
 	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Experimental: &config.Experimental{
 				PyDABs: config.PyDABs{
@@ -333,7 +333,7 @@ func TestFindExecPathFromEnvironmentWithWrongVersion(t *testing.T) {
 	ctx := context.Background()
 	m := &initialize{}
 	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target:    "whatever",
@@ -357,7 +357,7 @@ func TestFindExecPathFromEnvironmentWithCorrectVersionAndNoBinary(t *testing.T)
 	ctx := context.Background()
 	m := &initialize{}
 	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target:    "whatever",
@@ -380,7 +380,7 @@ func TestFindExecPathFromEnvironmentWithCorrectVersionAndBinary(t *testing.T) {
 	ctx := context.Background()
 	m := &initialize{}
 	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target:    "whatever",

bundle/deploy/terraform/interpolate.go

@@ -60,6 +60,8 @@ func (m *interpolateMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.D
 				path = dyn.NewPath(dyn.Key("databricks_schema")).Append(path[2:]...)
 			case dyn.Key("volumes"):
 				path = dyn.NewPath(dyn.Key("databricks_volume")).Append(path[2:]...)
+			case dyn.Key("clusters"):
+				path = dyn.NewPath(dyn.Key("databricks_cluster")).Append(path[2:]...)
 			default:
 				// Trigger "key not found" for unknown resource types.
 				return dyn.GetByPath(root, path)

bundle/deploy/terraform/interpolate_test.go

@@ -32,6 +32,7 @@ func TestInterpolate(t *testing.T) {
 								"other_registered_model": "${resources.registered_models.other_registered_model.id}",
 								"other_schema":           "${resources.schemas.other_schema.id}",
 								"other_volume":           "${resources.volumes.other_volume.id}",
+								"other_cluster":          "${resources.clusters.other_cluster.id}",
 							},
 							Tasks: []jobs.Task{
 								{
@@ -69,6 +70,7 @@ func TestInterpolate(t *testing.T) {
 	assert.Equal(t, "${databricks_registered_model.other_registered_model.id}", j.Tags["other_registered_model"])
 	assert.Equal(t, "${databricks_schema.other_schema.id}", j.Tags["other_schema"])
 	assert.Equal(t, "${databricks_volume.other_volume.id}", j.Tags["other_volume"])
+	assert.Equal(t, "${databricks_cluster.other_cluster.id}", j.Tags["other_cluster"])
 
 	m := b.Config.Resources.Models["my_model"]
 	assert.Equal(t, "my_model", m.Model.Name)

bundle/deploy/terraform/load_test.go

@@ -17,7 +17,7 @@ func TestLoadWithNoState(t *testing.T) {
 	}
 
 	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target: "whatever",

bundle/deploy/terraform/state_pull_test.go

@@ -32,7 +32,7 @@ func mockStateFilerForPull(t *testing.T, contents map[string]any, merr error) fi
 
 func statePullTestBundle(t *testing.T) *bundle.Bundle {
 	return &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target: "default",

bundle/deploy/terraform/state_push.go

@@ -47,6 +47,17 @@ func (l *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic
 	}
 	defer local.Close()
 
+	if !b.Config.Bundle.Force {
+		state, err := local.Stat()
+		if err != nil {
+			return diag.FromErr(err)
+		}
+
+		if state.Size() > deploy.MaxStateFileSize {
+			return diag.Errorf("Terraform state file size exceeds the maximum allowed size of %d bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag", deploy.MaxStateFileSize)
+		}
+	}
+
 	// Upload state file from local cache directory to filer.
 	cmdio.LogString(ctx, "Updating deployment state...")
 	log.Infof(ctx, "Writing local state file to remote state directory")

bundle/deploy/terraform/state_push_test.go

@@ -3,6 +3,7 @@ package terraform
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"testing"
 
@@ -29,7 +30,7 @@ func mockStateFilerForPush(t *testing.T, fn func(body io.Reader)) filer.Filer {
 
 func statePushTestBundle(t *testing.T) *bundle.Bundle {
 	return &bundle.Bundle{
-		RootPath: t.TempDir(),
+		BundleRootPath: t.TempDir(),
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target: "default",
@@ -59,3 +60,29 @@ func TestStatePush(t *testing.T) {
 	diags := bundle.Apply(ctx, b, m)
 	assert.NoError(t, diags.Error())
 }
+
+func TestStatePushLargeState(t *testing.T) {
+	mock := mockfiler.NewMockFiler(t)
+	m := &statePush{
+		identityFiler(mock),
+	}
+
+	ctx := context.Background()
+	b := statePushTestBundle(t)
+
+	largeState := map[string]any{}
+	for i := 0; i < 1000000; i++ {
+		largeState[fmt.Sprintf("field_%d", i)] = i
+	}
+
+	// Write a stale local state file.
+	writeLocalState(t, ctx, b, largeState)
+	diags := bundle.Apply(ctx, b, m)
+	assert.ErrorContains(t, diags.Error(), "Terraform state file size exceeds the maximum allowed size of 10485760 bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag")
+
+	// Force the write.
+	b = statePushTestBundle(t)
+	b.Config.Bundle.Force = true
+	diags = bundle.Apply(ctx, b, m)
+	assert.NoError(t, diags.Error())
+}

bundle/deploy/terraform/tfdyn/convert_cluster.go

@@ -0,0 +1,52 @@
+package tfdyn
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/cli/bundle/internal/tf/schema"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/databricks-sdk-go/service/compute"
+)
+
+func convertClusterResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) {
+	// Normalize the output value to the target schema.
+	vout, diags := convert.Normalize(compute.ClusterSpec{}, vin)
+	for _, diag := range diags {
+		log.Debugf(ctx, "cluster normalization diagnostic: %s", diag.Summary)
+	}
+
+	return vout, nil
+}
+
+type clusterConverter struct{}
+
+func (clusterConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error {
+	vout, err := convertClusterResource(ctx, vin)
+	if err != nil {
+		return err
+	}
+
+	// We always set no_wait as it allows DABs not to wait for cluster to be started.
+	vout, err = dyn.Set(vout, "no_wait", dyn.V(true))
+	if err != nil {
+		return err
+	}
+
+	// Add the converted resource to the output.
+	out.Cluster[key] = vout.AsAny()
+
+	// Configure permissions for this resource.
+	if permissions := convertPermissionsResource(ctx, vin); permissions != nil {
+		permissions.ClusterId = fmt.Sprintf("${databricks_cluster.%s.id}", key)
+		out.Permissions["cluster_"+key] = permissions
+	}
+
+	return nil
+}
+
+func init() {
+	registerConverter("clusters", clusterConverter{})
+}

bundle/deploy/terraform/tfdyn/convert_cluster_test.go

@@ -0,0 +1,97 @@
+package tfdyn
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/internal/tf/schema"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/databricks/databricks-sdk-go/service/compute"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConvertCluster(t *testing.T) {
+	var src = resources.Cluster{
+		ClusterSpec: &compute.ClusterSpec{
+			NumWorkers:   3,
+			SparkVersion: "13.3.x-scala2.12",
+			ClusterName:  "cluster",
+			SparkConf: map[string]string{
+				"spark.executor.memory": "2g",
+			},
+			AwsAttributes: &compute.AwsAttributes{
+				Availability: "ON_DEMAND",
+			},
+			AzureAttributes: &compute.AzureAttributes{
+				Availability: "SPOT",
+			},
+			DataSecurityMode: "USER_ISOLATION",
+			NodeTypeId:       "m5.xlarge",
+			Autoscale: &compute.AutoScale{
+				MinWorkers: 1,
+				MaxWorkers: 10,
+			},
+		},
+
+		Permissions: []resources.Permission{
+			{
+				Level:    "CAN_RUN",
+				UserName: "jack@gmail.com",
+			},
+			{
+				Level:                "CAN_MANAGE",
+				ServicePrincipalName: "sp",
+			},
+		},
+	}
+
+	vin, err := convert.FromTyped(src, dyn.NilValue)
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	out := schema.NewResources()
+	err = clusterConverter{}.Convert(ctx, "my_cluster", vin, out)
+	require.NoError(t, err)
+
+	cluster := out.Cluster["my_cluster"]
+	assert.Equal(t, map[string]any{
+		"num_workers":   int64(3),
+		"spark_version": "13.3.x-scala2.12",
+		"cluster_name":  "cluster",
+		"spark_conf": map[string]any{
+			"spark.executor.memory": "2g",
+		},
+		"aws_attributes": map[string]any{
+			"availability": "ON_DEMAND",
+		},
+		"azure_attributes": map[string]any{
+			"availability": "SPOT",
+		},
+		"data_security_mode": "USER_ISOLATION",
+		"no_wait":            true,
+		"node_type_id":       "m5.xlarge",
+		"autoscale": map[string]any{
+			"min_workers": int64(1),
+			"max_workers": int64(10),
+		},
+	}, cluster)
+
+	// Assert equality on the permissions
+	assert.Equal(t, &schema.ResourcePermissions{
+		ClusterId: "${databricks_cluster.my_cluster.id}",
+		AccessControl: []schema.ResourcePermissionsAccessControl{
+			{
+				PermissionLevel: "CAN_RUN",
+				UserName:        "jack@gmail.com",
+			},
+			{
+				PermissionLevel:      "CAN_MANAGE",
+				ServicePrincipalName: "sp",
+			},
+		},
+	}, out.Permissions["cluster_my_cluster"])
+
+}

bundle/deploy/terraform/tfdyn/convert_job.go

@@ -3,6 +3,7 @@ package tfdyn
 import (
 	"context"
 	"fmt"
+	"sort"
 
 	"github.com/databricks/cli/bundle/internal/tf/schema"
 	"github.com/databricks/cli/libs/dyn"
@@ -19,8 +20,38 @@ func convertJobResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) {
 		log.Debugf(ctx, "job normalization diagnostic: %s", diag.Summary)
 	}
 
+	// Sort the tasks of each job in the bundle by task key. Sorting
+	// the task keys ensures that the diff computed by terraform is correct and avoids
+	// recreates. For more details see the NOTE at
+	// https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/job#example-usage
+	// and https://github.com/databricks/terraform-provider-databricks/issues/4011
+	// and https://github.com/databricks/cli/pull/1776
+	vout := vin
+	var err error
+	tasks, ok := vin.Get("tasks").AsSequence()
+	if ok {
+		sort.Slice(tasks, func(i, j int) bool {
+			// We sort the tasks by their task key. Tasks without task keys are ordered
+			// before tasks with task keys. We do not error for those tasks
+			// since presence of a task_key is validated for in the Jobs backend.
+			tk1, ok := tasks[i].Get("task_key").AsString()
+			if !ok {
+				return true
+			}
+			tk2, ok := tasks[j].Get("task_key").AsString()
+			if !ok {
+				return false
+			}
+			return tk1 < tk2
+		})
+		vout, err = dyn.Set(vin, "tasks", dyn.V(tasks))
+		if err != nil {
+			return dyn.InvalidValue, err
+		}
+	}
+
 	// Modify top-level keys.
-	vout, err := renameKeys(vin, map[string]string{
+	vout, err = renameKeys(vout, map[string]string{
 		"tasks":        "task",
 		"job_clusters": "job_cluster",
 		"parameters":   "parameter",

bundle/deploy/terraform/tfdyn/convert_job_test.go

@@ -42,8 +42,8 @@ func TestConvertJob(t *testing.T) {
 			},
 			Tasks: []jobs.Task{
 				{
-					TaskKey:       "task_key",
+					TaskKey:       "task_key_b",
-					JobClusterKey: "job_cluster_key",
+					JobClusterKey: "job_cluster_key_b",
 					Libraries: []compute.Library{
 						{
 							Pypi: &compute.PythonPyPiLibrary{
@@ -55,6 +55,17 @@ func TestConvertJob(t *testing.T) {
 						},
 					},
 				},
+				{
+					TaskKey:       "task_key_a",
+					JobClusterKey: "job_cluster_key_a",
+				},
+				{
+					TaskKey:       "task_key_c",
+					JobClusterKey: "job_cluster_key_c",
+				},
+				{
+					Description: "missing task key 😱",
+				},
 			},
 		},
 		Permissions: []resources.Permission{
@@ -100,8 +111,15 @@ func TestConvertJob(t *testing.T) {
 		},
 		"task": []any{
 			map[string]any{
-				"task_key":        "task_key",
+				"description": "missing task key 😱",
-				"job_cluster_key": "job_cluster_key",
+			},
+			map[string]any{
+				"task_key":        "task_key_a",
+				"job_cluster_key": "job_cluster_key_a",
+			},
+			map[string]any{
+				"task_key":        "task_key_b",
+				"job_cluster_key": "job_cluster_key_b",
 				"library": []any{
 					map[string]any{
 						"pypi": map[string]any{
@@ -113,6 +131,10 @@ func TestConvertJob(t *testing.T) {
 					},
 				},
 			},
+			map[string]any{
+				"task_key":        "task_key_c",
+				"job_cluster_key": "job_cluster_key_c",
+			},
 		},
 	}, out.Job["my_job"])