Merge 977ba12f25 into b323703c1b

## Changes
This PR adds a warning validating that the configuration for a single
node cluster is valid for interactive, job, job-task, and pipeline
clusters.

Note: We skip the validation if a cluster policy is configured because
the policy is likely to configure `spark_conf` / `custom_tags` itself.

Note: Terrform originally only had validation for interactive, job, and
job-task clusters. This PR adding the validation for pipeline clusters
as well is new.

This PR follows the same logic as we used to have in Terraform. The
validation was removed from Terraform because we had no way to demote
the error to a warning:
https://github.com/databricks/terraform-provider-databricks/pull/4222

### Background
Single-node clusters require `spark_conf` and `custom_tags` to be
correctly set in the cluster definition for them to function optimally.
The cluster will be created even if incorrectly configured, but its
performance will not be great.

For example, if both `spark_conf` and `custom_tags` are not set and
`num_workers` is 0, then only the driver process will be launched on the
cluster compute instance thus leading to sub-optimal utilization of
available compute resources and no parallelization across worker
processes when processing a spark query.

### Issue

This PR addresses some issues reported in
https://github.com/databricks/cli/issues/1546

## Tests
Unit tests and manually.

Example output of the warning:
```
➜  bundle-playground git:(master) ✗ cli bundle validate
Warning: Single node cluster is not correctly configured
  at resources.pipelines.bar.clusters[0]
  in databricks.yml:29:11

num_workers should be 0 only for single-node clusters. To create a
valid single node cluster please ensure that the following properties
are correctly set in the cluster specification:

  spark_conf:
    spark.databricks.cluster.profile: singleNode
    spark.master: local[*]

  custom_tags:
    ResourceClass: SingleNode
  

Name: foobar
Target: default
Workspace:
  User: shreyas.goenka@databricks.com
  Path: /Workspace/Users/shreyas.goenka@databricks.com/.bundle/foobar/default

Found 1 warning
```

.codegen.json

@@ -5,8 +5,7 @@
     },
     "batch": {
         ".codegen/cmds-workspace.go.tmpl": "cmd/workspace/cmd.go",
-        ".codegen/cmds-account.go.tmpl": "cmd/account/cmd.go",
-        ".codegen/lookup.go.tmpl": "bundle/config/variable/lookup.go"
+        ".codegen/cmds-account.go.tmpl": "cmd/account/cmd.go"
     },
     "toolchain": {
         "required": ["go"],

.codegen/lookup.go.tmpl

@@ -1,134 +0,0 @@
-// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT.
-
-package variable
-
-{{ $allowlist :=
-	list
-		"alerts"
-		"clusters"
-		"cluster-policies"
-		"clusters"
-		"dashboards"
-		"instance-pools"
-		"jobs"
-		"metastores"
-		"pipelines"
-		"service-principals"
-		"queries"
-		"warehouses"
-}}
-
-{{	$customField :=
-		dict
-			"service-principals" "ApplicationId"
-}}
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/databricks/databricks-sdk-go"
-)
-
-type Lookup struct {
-	{{range .Services -}}
-	{{- if in $allowlist .KebabName -}}
-	{{.Singular.PascalName}} string `json:"{{.Singular.SnakeName}},omitempty"`
-
-	{{end}}
-	{{- end}}
-}
-
-func LookupFromMap(m map[string]any) *Lookup {
-	l := &Lookup{}
-	{{range .Services -}}
-	{{- if in $allowlist .KebabName -}}
-	if v, ok := m["{{.Singular.SnakeName}}"]; ok {
-		l.{{.Singular.PascalName}} = v.(string)
-	}
-	{{end -}}
-	{{- end}}
-	return l
-}
-
-func (l *Lookup) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
-	if err := l.validate(); err != nil {
-		return "", err
-	}
-
-	r := allResolvers()
-	{{range .Services -}}
-	{{- if in $allowlist .KebabName -}}
-	if l.{{.Singular.PascalName}} != "" {
-		return r.{{.Singular.PascalName}}(ctx, w, l.{{.Singular.PascalName}})
-	}
-	{{end -}}
-	{{- end}}
-
-	return "", fmt.Errorf("no valid lookup fields provided")
-}
-
-func (l *Lookup) String() string {
-	{{range .Services -}}
-	{{- if in $allowlist .KebabName -}}
-	if l.{{.Singular.PascalName}} != "" {
-		return fmt.Sprintf("{{.Singular.KebabName}}: %s", l.{{.Singular.PascalName}})
-	}
-	{{end -}}
-	{{- end}}
-	return ""
-}
-
-func (l *Lookup) validate() error {
-	// Validate that only one field is set
-	count := 0
-	{{range .Services -}}
-	{{- if in $allowlist .KebabName -}}
-	if l.{{.Singular.PascalName}} != "" {
-		count++
-	}
-	{{end -}}
-	{{- end}}
-
-	if count != 1 {
-		return fmt.Errorf("exactly one lookup field must be provided")
-	}
-
-	if strings.Contains(l.String(), "${var") {
-		return fmt.Errorf("lookup fields cannot contain variable references")
-	}
-
-	return nil
-}
-
-
-type resolverFunc func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error)
-type resolvers struct {
-	{{range .Services -}}
-	{{- if in $allowlist .KebabName -}}
-	{{.Singular.PascalName}} resolverFunc
-	{{end -}}
-	{{- end}}
-}
-
-func allResolvers() *resolvers {
-	r := &resolvers{}
-	{{range .Services -}}
-	{{- if in $allowlist .KebabName -}}
-	r.{{.Singular.PascalName}} = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["{{.Singular.PascalName}}"]
-		if ok {
-			return fn(ctx, w, name)
-		}
-		entity, err := w.{{.PascalName}}.GetBy{{range .NamedIdMap.NamePath}}{{.PascalName}}{{end}}(ctx, name)
-		if err != nil {
-			return "", err
-		}
-
-		return fmt.Sprint(entity.{{ getOrDefault $customField .KebabName ((index .NamedIdMap.IdPath 0).PascalName) }}), nil
-	}
-	{{end -}}
-	{{- end}}
-
-	return r
-}

.gitattributes

@@ -1,4 +1,3 @@
-bundle/config/variable/lookup.go linguist-generated=true
 cmd/account/access-control/access-control.go linguist-generated=true
 cmd/account/billable-usage/billable-usage.go linguist-generated=true
 cmd/account/budgets/budgets.go linguist-generated=true

CHANGELOG.md

@@ -1,5 +1,28 @@
 # Version changelog
 
+## [Release] Release v0.235.0
+
+**Note:** the `bundle generate` command now uses the `.<resource-type>.yml`
+sub-extension for the configuration files it writes. Existing configuration
+files that do not use this sub-extension are renamed to include it.
+
+Bundles:
+ * Make `TableName` field part of quality monitor schema ([#1903](https://github.com/databricks/cli/pull/1903)).
+ * Do not prepend paths starting with ~ or variable reference ([#1905](https://github.com/databricks/cli/pull/1905)).
+ * Fix workspace extensions filer accidentally reading notebooks ([#1891](https://github.com/databricks/cli/pull/1891)).
+ * Fix template initialization when running on Databricks ([#1912](https://github.com/databricks/cli/pull/1912)).
+ * Source-linked deployments for bundles in the workspace ([#1884](https://github.com/databricks/cli/pull/1884)).
+ * Added integration test to deploy bundle to /Shared root path ([#1914](https://github.com/databricks/cli/pull/1914)).
+ * Update filenames used by bundle generate to use `.<resource-type>.yml` ([#1901](https://github.com/databricks/cli/pull/1901)).
+
+Internal:
+ * Extract functionality to detect if the CLI is running on DBR ([#1889](https://github.com/databricks/cli/pull/1889)).
+ * Consolidate test helpers for `io/fs` ([#1906](https://github.com/databricks/cli/pull/1906)).
+ * Use `fs.FS` interface to read template ([#1910](https://github.com/databricks/cli/pull/1910)).
+ * Use `filer.Filer` to write template instantiation ([#1911](https://github.com/databricks/cli/pull/1911)).
+
+
+
 ## [Release] Release v0.234.0
 
 Bundles:

bundle/config/mutator/apply_presets.go

@@ -225,9 +225,21 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 	if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) {
 		isDatabricksWorkspace := dbr.RunsOnRuntime(ctx) && strings.HasPrefix(b.SyncRootPath, "/Workspace/")
 		if !isDatabricksWorkspace {
+			target := b.Config.Bundle.Target
+			path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment"))
+			diags = diags.Append(
+				diag.Diagnostic{
+					Severity: diag.Warning,
+					Summary:  "source-linked deployment is available only in the Databricks Workspace",
+					Paths: []dyn.Path{
+						path,
+					},
+					Locations: b.Config.GetLocations(path[2:].String()),
+				},
+			)
+
 			disabled := false
 			b.Config.Presets.SourceLinkedDeployment = &disabled
-			diags = diags.Extend(diag.Warningf("source-linked deployment is available only in the Databricks Workspace"))
 		}
 	}
 

bundle/config/mutator/apply_presets_test.go

@@ -9,7 +9,9 @@ import (
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/mutator"
 	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/internal/bundletest"
 	"github.com/databricks/cli/libs/dbr"
+	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/databricks-sdk-go/service/catalog"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/stretchr/testify/require"
@@ -435,6 +437,7 @@ func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
 				},
 			}
 
+			bundletest.SetLocation(b, "presets.source_linked_deployment", []dyn.Location{{File: "databricks.yml"}})
 			diags := bundle.Apply(tt.ctx, b, mutator.ApplyPresets())
 			if diags.HasError() {
 				t.Fatalf("unexpected error: %v", diags)
@@ -442,6 +445,7 @@ func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
 
 			if tt.expectedWarning != "" {
 				require.Equal(t, tt.expectedWarning, diags[0].Summary)
+				require.NotEmpty(t, diags[0].Locations)
 			}
 
 			require.Equal(t, tt.expectedValue, b.Config.Presets.SourceLinkedDeployment)

bundle/config/mutator/translate_paths.go

@@ -126,8 +126,34 @@ func (t *translateContext) rewritePath(
 func (t *translateContext) translateNotebookPath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
 	nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, filepath.ToSlash(localRelPath))
 	if errors.Is(err, fs.ErrNotExist) {
+		if filepath.Ext(localFullPath) != notebook.ExtensionNone {
 			return "", fmt.Errorf("notebook %s not found", literal)
 		}
+
+		extensions := []string{
+			notebook.ExtensionPython,
+			notebook.ExtensionR,
+			notebook.ExtensionScala,
+			notebook.ExtensionSql,
+			notebook.ExtensionJupyter,
+		}
+
+		// Check whether a file with a notebook extension already exists. This
+		// way we can provide a more targeted error message.
+		for _, ext := range extensions {
+			literalWithExt := literal + ext
+			localRelPathWithExt := filepath.ToSlash(localRelPath + ext)
+			if _, err := fs.Stat(t.b.SyncRoot, localRelPathWithExt); err == nil {
+				return "", fmt.Errorf(`notebook %s not found. Did you mean %s?
+Local notebook references are expected to contain one of the following
+file extensions: [%s]`, literal, literalWithExt, strings.Join(extensions, ", "))
+			}
+		}
+
+		// Return a generic error message if no matching possible file is found.
+		return "", fmt.Errorf(`notebook %s not found. Local notebook references are expected
+to contain one of the following file extensions: [%s]`, literal, strings.Join(extensions, ", "))
+	}
 	if err != nil {
 		return "", fmt.Errorf("unable to determine if %s is a notebook: %w", localFullPath, err)
 	}

bundle/config/mutator/translate_paths_test.go

@@ -2,6 +2,7 @@ package mutator_test
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -508,6 +509,59 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) {
 	assert.EqualError(t, diags.Error(), "notebook ./doesnt_exist.py not found")
 }
 
+func TestPipelineNotebookDoesNotExistErrorWithoutExtension(t *testing.T) {
+	for _, ext := range []string{
+		".py",
+		".r",
+		".scala",
+		".sql",
+		".ipynb",
+		"",
+	} {
+		t.Run("case_"+ext, func(t *testing.T) {
+			dir := t.TempDir()
+
+			if ext != "" {
+				touchEmptyFile(t, filepath.Join(dir, "foo"+ext))
+			}
+
+			b := &bundle.Bundle{
+				SyncRootPath: dir,
+				SyncRoot:     vfs.MustNew(dir),
+				Config: config.Root{
+					Resources: config.Resources{
+						Pipelines: map[string]*resources.Pipeline{
+							"pipeline": {
+								PipelineSpec: &pipelines.PipelineSpec{
+									Libraries: []pipelines.PipelineLibrary{
+										{
+											Notebook: &pipelines.NotebookLibrary{
+												Path: "./foo",
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			}
+
+			bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}})
+			diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
+
+			if ext == "" {
+				assert.EqualError(t, diags.Error(), `notebook ./foo not found. Local notebook references are expected
+to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb]`)
+			} else {
+				assert.EqualError(t, diags.Error(), fmt.Sprintf(`notebook ./foo not found. Did you mean ./foo%s?
+Local notebook references are expected to contain one of the following
+file extensions: [.py, .r, .scala, .sql, .ipynb]`, ext))
+			}
+		})
+	}
+}
+
 func TestPipelineFileDoesNotExistError(t *testing.T) {
 	dir := t.TempDir()
 

bundle/config/validate/files_to_sync.go

@@ -21,6 +21,12 @@ func (v *filesToSync) Name() string {
 }
 
 func (v *filesToSync) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics {
+	// The user may be intentional about not synchronizing any files.
+	// In this case, we should not show any warnings.
+	if len(rb.Config().Sync.Paths) == 0 {
+		return nil
+	}
+
 	sync, err := files.GetSync(ctx, rb)
 	if err != nil {
 		return diag.FromErr(err)
@@ -31,6 +37,7 @@ func (v *filesToSync) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.
 		return diag.FromErr(err)
 	}
 
+	// If there are files to sync, we don't need to show any warnings.
 	if len(fl) != 0 {
 		return nil
 	}

bundle/config/validate/files_to_sync_test.go

@@ -0,0 +1,105 @@
+package validate
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/internal/testutil"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/vfs"
+	sdkconfig "github.com/databricks/databricks-sdk-go/config"
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/service/iam"
+	"github.com/databricks/databricks-sdk-go/service/workspace"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFilesToSync_NoPaths(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Sync: config.Sync{
+				Paths: []string{},
+			},
+		},
+	}
+
+	ctx := context.Background()
+	rb := bundle.ReadOnly(b)
+	diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync())
+	assert.Empty(t, diags)
+}
+
+func setupBundleForFilesToSyncTest(t *testing.T) *bundle.Bundle {
+	dir := t.TempDir()
+
+	testutil.Touch(t, dir, "file1")
+	testutil.Touch(t, dir, "file2")
+
+	b := &bundle.Bundle{
+		BundleRootPath: dir,
+		BundleRoot:     vfs.MustNew(dir),
+		SyncRootPath:   dir,
+		SyncRoot:       vfs.MustNew(dir),
+		Config: config.Root{
+			Bundle: config.Bundle{
+				Target: "default",
+			},
+			Workspace: config.Workspace{
+				FilePath: "/this/doesnt/matter",
+				CurrentUser: &config.User{
+					User: &iam.User{},
+				},
+			},
+			Sync: config.Sync{
+				// Paths are relative to [SyncRootPath].
+				Paths: []string{"."},
+			},
+		},
+	}
+
+	m := mocks.NewMockWorkspaceClient(t)
+	m.WorkspaceClient.Config = &sdkconfig.Config{
+		Host: "https://foo.com",
+	}
+
+	// The initialization logic in [sync.New] performs a check on the destination path.
+	// Removing this check at initialization time is tbd...
+	m.GetMockWorkspaceAPI().EXPECT().GetStatusByPath(mock.Anything, "/this/doesnt/matter").Return(&workspace.ObjectInfo{
+		ObjectType: workspace.ObjectTypeDirectory,
+	}, nil)
+
+	b.SetWorkpaceClient(m.WorkspaceClient)
+	return b
+}
+
+func TestFilesToSync_EverythingIgnored(t *testing.T) {
+	b := setupBundleForFilesToSyncTest(t)
+
+	// Ignore all files.
+	testutil.WriteFile(t, "*\n.*\n", b.BundleRootPath, ".gitignore")
+
+	ctx := context.Background()
+	rb := bundle.ReadOnly(b)
+	diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync())
+	require.Equal(t, 1, len(diags))
+	assert.Equal(t, diag.Warning, diags[0].Severity)
+	assert.Equal(t, "There are no files to sync, please check your .gitignore", diags[0].Summary)
+}
+
+func TestFilesToSync_EverythingExcluded(t *testing.T) {
+	b := setupBundleForFilesToSyncTest(t)
+
+	// Exclude all files.
+	b.Config.Sync.Exclude = []string{"*"}
+
+	ctx := context.Background()
+	rb := bundle.ReadOnly(b)
+	diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync())
+	require.Equal(t, 1, len(diags))
+	assert.Equal(t, diag.Warning, diags[0].Severity)
+	assert.Equal(t, "There are no files to sync, please check your .gitignore and sync.exclude configuration", diags[0].Summary)
+}

bundle/config/validate/single_node_cluster.go

@@ -0,0 +1,137 @@
+package validate
+
+import (
+	"context"
+	"strings"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/databricks/cli/libs/log"
+)
+
+// Validates that any single node clusters defined in the bundle are correctly configured.
+func SingleNodeCluster() bundle.ReadOnlyMutator {
+	return &singleNodeCluster{}
+}
+
+type singleNodeCluster struct{}
+
+func (m *singleNodeCluster) Name() string {
+	return "validate:SingleNodeCluster"
+}
+
+const singleNodeWarningDetail = `num_workers should be 0 only for single-node clusters. To create a
+valid single node cluster please ensure that the following properties
+are correctly set in the cluster specification:
+
+  spark_conf:
+    spark.databricks.cluster.profile: singleNode
+    spark.master: local[*]
+
+  custom_tags:
+    ResourceClass: SingleNode
+  `
+
+const singleNodeWarningSummary = `Single node cluster is not correctly configured`
+
+func showSingleNodeClusterWarning(ctx context.Context, v dyn.Value) bool {
+	// Check if the user has explicitly set the num_workers to 0. Skip the warning
+	// if that's not the case.
+	numWorkers, ok := v.Get("num_workers").AsInt()
+	if !ok || numWorkers > 0 {
+		return false
+	}
+
+	// Convenient type that contains the common fields from compute.ClusterSpec and
+	// pipelines.PipelineCluster that we are interested in.
+	type ClusterConf struct {
+		SparkConf  map[string]string `json:"spark_conf"`
+		CustomTags map[string]string `json:"custom_tags"`
+		PolicyId   string            `json:"policy_id"`
+	}
+
+	conf := &ClusterConf{}
+	err := convert.ToTyped(conf, v)
+	if err != nil {
+		return false
+	}
+
+	// If the policy id is set, we don't want to show the warning. This is because
+	// the user might have configured `spark_conf` and `custom_tags` correctly
+	// in their cluster policy.
+	if conf.PolicyId != "" {
+		return false
+	}
+
+	profile, ok := conf.SparkConf["spark.databricks.cluster.profile"]
+	if !ok {
+		log.Debugf(ctx, "spark_conf spark.databricks.cluster.profile not found in single-node cluster spec")
+		return true
+	}
+	if profile != "singleNode" {
+		log.Debugf(ctx, "spark_conf spark.databricks.cluster.profile is not singleNode in single-node cluster spec: %s", profile)
+		return true
+	}
+
+	master, ok := conf.SparkConf["spark.master"]
+	if !ok {
+		log.Debugf(ctx, "spark_conf spark.master not found in single-node cluster spec")
+		return true
+	}
+	if !strings.HasPrefix(master, "local") {
+		log.Debugf(ctx, "spark_conf spark.master does not start with local in single-node cluster spec: %s", master)
+		return true
+	}
+
+	resourceClass, ok := conf.CustomTags["ResourceClass"]
+	if !ok {
+		log.Debugf(ctx, "custom_tag ResourceClass not found in single-node cluster spec")
+		return true
+	}
+	if resourceClass != "SingleNode" {
+		log.Debugf(ctx, "custom_tag ResourceClass is not SingleNode in single-node cluster spec: %s", resourceClass)
+		return true
+	}
+
+	return false
+}
+
+func (m *singleNodeCluster) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics {
+	diags := diag.Diagnostics{}
+
+	patterns := []dyn.Pattern{
+		// Interactive clusters
+		dyn.NewPattern(dyn.Key("resources"), dyn.Key("clusters"), dyn.AnyKey()),
+		// Job clusters
+		dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), dyn.Key("job_clusters"), dyn.AnyIndex(), dyn.Key("new_cluster")),
+		// Job task clusters
+		dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), dyn.Key("tasks"), dyn.AnyIndex(), dyn.Key("new_cluster")),
+		// Job for each task clusters
+		dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), dyn.Key("tasks"), dyn.AnyIndex(), dyn.Key("for_each_task"), dyn.Key("task"), dyn.Key("new_cluster")),
+		// Pipeline clusters
+		dyn.NewPattern(dyn.Key("resources"), dyn.Key("pipelines"), dyn.AnyKey(), dyn.Key("clusters"), dyn.AnyIndex()),
+	}
+
+	for _, p := range patterns {
+		_, err := dyn.MapByPattern(rb.Config().Value(), p, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+			warning := diag.Diagnostic{
+				Severity:  diag.Warning,
+				Summary:   singleNodeWarningSummary,
+				Detail:    singleNodeWarningDetail,
+				Locations: v.Locations(),
+				Paths:     []dyn.Path{p},
+			}
+
+			if showSingleNodeClusterWarning(ctx, v) {
+				diags = append(diags, warning)
+			}
+			return v, nil
+		})
+		if err != nil {
+			log.Debugf(ctx, "Error while applying single node cluster validation: %s", err)
+		}
+	}
+	return diags
+}

bundle/config/validate/single_node_cluster_test.go

@@ -0,0 +1,566 @@
+package validate
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/internal/bundletest"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/databricks-sdk-go/service/compute"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/databricks/databricks-sdk-go/service/pipelines"
+	"github.com/stretchr/testify/assert"
+)
+
+func failCases() []struct {
+	name       string
+	sparkConf  map[string]string
+	customTags map[string]string
+} {
+	return []struct {
+		name       string
+		sparkConf  map[string]string
+		customTags map[string]string
+	}{
+		{
+			name: "no tags or conf",
+		},
+		{
+			name: "no tags",
+			sparkConf: map[string]string{
+				"spark.databricks.cluster.profile": "singleNode",
+				"spark.master":                     "local[*]",
+			},
+		},
+		{
+			name:       "no conf",
+			customTags: map[string]string{"ResourceClass": "SingleNode"},
+		},
+		{
+			name: "invalid spark cluster profile",
+			sparkConf: map[string]string{
+				"spark.databricks.cluster.profile": "invalid",
+				"spark.master":                     "local[*]",
+			},
+			customTags: map[string]string{"ResourceClass": "SingleNode"},
+		},
+		{
+			name: "invalid spark.master",
+			sparkConf: map[string]string{
+				"spark.databricks.cluster.profile": "singleNode",
+				"spark.master":                     "invalid",
+			},
+			customTags: map[string]string{"ResourceClass": "SingleNode"},
+		},
+		{
+			name: "invalid tags",
+			sparkConf: map[string]string{
+				"spark.databricks.cluster.profile": "singleNode",
+				"spark.master":                     "local[*]",
+			},
+			customTags: map[string]string{"ResourceClass": "invalid"},
+		},
+		{
+			name: "missing ResourceClass tag",
+			sparkConf: map[string]string{
+				"spark.databricks.cluster.profile": "singleNode",
+				"spark.master":                     "local[*]",
+			},
+			customTags: map[string]string{"what": "ever"},
+		},
+		{
+			name: "missing spark.master",
+			sparkConf: map[string]string{
+				"spark.databricks.cluster.profile": "singleNode",
+			},
+			customTags: map[string]string{"ResourceClass": "SingleNode"},
+		},
+		{
+			name: "missing spark.databricks.cluster.profile",
+			sparkConf: map[string]string{
+				"spark.master": "local[*]",
+			},
+			customTags: map[string]string{"ResourceClass": "SingleNode"},
+		},
+	}
+}
+
+func TestValidateSingleNodeClusterFailForInteractiveClusters(t *testing.T) {
+	ctx := context.Background()
+
+	for _, tc := range failCases() {
+		t.Run(tc.name, func(t *testing.T) {
+			b := &bundle.Bundle{
+				Config: config.Root{
+					Resources: config.Resources{
+						Clusters: map[string]*resources.Cluster{
+							"foo": {
+								ClusterSpec: &compute.ClusterSpec{
+									SparkConf:  tc.sparkConf,
+									CustomTags: tc.customTags,
+								},
+							},
+						},
+					},
+				},
+			}
+
+			bundletest.SetLocation(b, "resources.clusters.foo", []dyn.Location{{File: "a.yml", Line: 1, Column: 1}})
+
+			// We can't set num_workers to 0 explicitly in the typed configuration.
+			// Do it on the dyn.Value directly.
+			bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
+				return dyn.Set(v, "resources.clusters.foo.num_workers", dyn.V(0))
+			})
+			diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
+			assert.Equal(t, diag.Diagnostics{
+				{
+					Severity:  diag.Warning,
+					Summary:   singleNodeWarningSummary,
+					Detail:    singleNodeWarningDetail,
+					Locations: []dyn.Location{{File: "a.yml", Line: 1, Column: 1}},
+					Paths:     []dyn.Path{dyn.NewPath(dyn.Key("resources"), dyn.Key("clusters"), dyn.Key("foo"))},
+				},
+			}, diags)
+		})
+	}
+}
+
+func TestValidateSingleNodeClusterFailForJobClusters(t *testing.T) {
+	ctx := context.Background()
+
+	for _, tc := range failCases() {
+		t.Run(tc.name, func(t *testing.T) {
+			b := &bundle.Bundle{
+				Config: config.Root{
+					Resources: config.Resources{
+						Jobs: map[string]*resources.Job{
+							"foo": {
+								JobSettings: &jobs.JobSettings{
+									JobClusters: []jobs.JobCluster{
+										{
+											NewCluster: compute.ClusterSpec{
+												ClusterName: "my_cluster",
+												SparkConf:   tc.sparkConf,
+												CustomTags:  tc.customTags,
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			}
+
+			bundletest.SetLocation(b, "resources.jobs.foo.job_clusters[0].new_cluster", []dyn.Location{{File: "b.yml", Line: 1, Column: 1}})
+
+			// We can't set num_workers to 0 explicitly in the typed configuration.
+			// Do it on the dyn.Value directly.
+			bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
+				return dyn.Set(v, "resources.jobs.foo.job_clusters[0].new_cluster.num_workers", dyn.V(0))
+			})
+
+			diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
+			assert.Equal(t, diag.Diagnostics{
+				{
+					Severity:  diag.Warning,
+					Summary:   singleNodeWarningSummary,
+					Detail:    singleNodeWarningDetail,
+					Locations: []dyn.Location{{File: "b.yml", Line: 1, Column: 1}},
+					Paths:     []dyn.Path{dyn.MustPathFromString("resources.jobs.foo.job_clusters[0].new_cluster")},
+				},
+			}, diags)
+
+		})
+	}
+}
+
+func TestValidateSingleNodeClusterFailForJobTaskClusters(t *testing.T) {
+	ctx := context.Background()
+
+	for _, tc := range failCases() {
+		t.Run(tc.name, func(t *testing.T) {
+			b := &bundle.Bundle{
+				Config: config.Root{
+					Resources: config.Resources{
+						Jobs: map[string]*resources.Job{
+							"foo": {
+								JobSettings: &jobs.JobSettings{
+									Tasks: []jobs.Task{
+										{
+											NewCluster: &compute.ClusterSpec{
+												ClusterName: "my_cluster",
+												SparkConf:   tc.sparkConf,
+												CustomTags:  tc.customTags,
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			}
+
+			bundletest.SetLocation(b, "resources.jobs.foo.tasks[0].new_cluster", []dyn.Location{{File: "c.yml", Line: 1, Column: 1}})
+
+			// We can't set num_workers to 0 explicitly in the typed configuration.
+			// Do it on the dyn.Value directly.
+			bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
+				return dyn.Set(v, "resources.jobs.foo.tasks[0].new_cluster.num_workers", dyn.V(0))
+			})
+
+			diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
+			assert.Equal(t, diag.Diagnostics{
+				{
+					Severity:  diag.Warning,
+					Summary:   singleNodeWarningSummary,
+					Detail:    singleNodeWarningDetail,
+					Locations: []dyn.Location{{File: "c.yml", Line: 1, Column: 1}},
+					Paths:     []dyn.Path{dyn.MustPathFromString("resources.jobs.foo.tasks[0].new_cluster")},
+				},
+			}, diags)
+		})
+	}
+}
+
+func TestValidateSingleNodeClusterFailForPipelineClusters(t *testing.T) {
+	ctx := context.Background()
+
+	for _, tc := range failCases() {
+		t.Run(tc.name, func(t *testing.T) {
+			b := &bundle.Bundle{
+				Config: config.Root{
+					Resources: config.Resources{
+						Pipelines: map[string]*resources.Pipeline{
+							"foo": {
+								PipelineSpec: &pipelines.PipelineSpec{
+									Clusters: []pipelines.PipelineCluster{
+										{
+											SparkConf:  tc.sparkConf,
+											CustomTags: tc.customTags,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			}
+
+			bundletest.SetLocation(b, "resources.pipelines.foo.clusters[0]", []dyn.Location{{File: "d.yml", Line: 1, Column: 1}})
+
+			// We can't set num_workers to 0 explicitly in the typed configuration.
+			// Do it on the dyn.Value directly.
+			bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
+				return dyn.Set(v, "resources.pipelines.foo.clusters[0].num_workers", dyn.V(0))
+			})
+
+			diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
+			assert.Equal(t, diag.Diagnostics{
+				{
+					Severity:  diag.Warning,
+					Summary:   singleNodeWarningSummary,
+					Detail:    singleNodeWarningDetail,
+					Locations: []dyn.Location{{File: "d.yml", Line: 1, Column: 1}},
+					Paths:     []dyn.Path{dyn.MustPathFromString("resources.pipelines.foo.clusters[0]")},
+				},
+			}, diags)
+		})
+	}
+}
+
+func TestValidateSingleNodeClusterFailForJobForEachTaskCluster(t *testing.T) {
+	ctx := context.Background()
+
+	for _, tc := range failCases() {
+		t.Run(tc.name, func(t *testing.T) {
+			b := &bundle.Bundle{
+				Config: config.Root{
+					Resources: config.Resources{
+						Jobs: map[string]*resources.Job{
+							"foo": {
+								JobSettings: &jobs.JobSettings{
+									Tasks: []jobs.Task{
+										{
+											ForEachTask: &jobs.ForEachTask{
+												Task: jobs.Task{
+													NewCluster: &compute.ClusterSpec{
+														ClusterName: "my_cluster",
+														SparkConf:   tc.sparkConf,
+														CustomTags:  tc.customTags,
+													},
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			}
+
+			bundletest.SetLocation(b, "resources.jobs.foo.tasks[0].for_each_task.task.new_cluster", []dyn.Location{{File: "e.yml", Line: 1, Column: 1}})
+
+			// We can't set num_workers to 0 explicitly in the typed configuration.
+			// Do it on the dyn.Value directly.
+			bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
+				return dyn.Set(v, "resources.jobs.foo.tasks[0].for_each_task.task.new_cluster.num_workers", dyn.V(0))
+			})
+
+			diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
+			assert.Equal(t, diag.Diagnostics{
+				{
+					Severity:  diag.Warning,
+					Summary:   singleNodeWarningSummary,
+					Detail:    singleNodeWarningDetail,
+					Locations: []dyn.Location{{File: "e.yml", Line: 1, Column: 1}},
+					Paths:     []dyn.Path{dyn.MustPathFromString("resources.jobs.foo.tasks[0].for_each_task.task.new_cluster")},
+				},
+			}, diags)
+		})
+	}
+}
+
+func passCases() []struct {
+	name       string
+	numWorkers *int
+	sparkConf  map[string]string
+	customTags map[string]string
+	policyId   string
+} {
+	zero := 0
+	one := 1
+
+	return []struct {
+		name       string
+		numWorkers *int
+		sparkConf  map[string]string
+		customTags map[string]string
+		policyId   string
+	}{
+		{
+			name: "single node cluster",
+			sparkConf: map[string]string{
+				"spark.databricks.cluster.profile": "singleNode",
+				"spark.master":                     "local[*]",
+			},
+			customTags: map[string]string{
+				"ResourceClass": "SingleNode",
+			},
+			numWorkers: &zero,
+		},
+		{
+			name:       "num workers is not zero",
+			numWorkers: &one,
+		},
+		{
+			name: "num workers is not set",
+		},
+		{
+			name:       "policy id is not empty",
+			policyId:   "policy-abc",
+			numWorkers: &zero,
+		},
+	}
+}
+
+func TestValidateSingleNodeClusterPassInteractiveClusters(t *testing.T) {
+	ctx := context.Background()
+
+	for _, tc := range passCases() {
+		t.Run(tc.name, func(t *testing.T) {
+			b := &bundle.Bundle{
+				Config: config.Root{
+					Resources: config.Resources{
+						Clusters: map[string]*resources.Cluster{
+							"foo": {
+								ClusterSpec: &compute.ClusterSpec{
+									SparkConf:  tc.sparkConf,
+									CustomTags: tc.customTags,
+									PolicyId:   tc.policyId,
+								},
+							},
+						},
+					},
+				},
+			}
+
+			if tc.numWorkers != nil {
+				bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
+					return dyn.Set(v, "resources.clusters.foo.num_workers", dyn.V(*tc.numWorkers))
+				})
+			}
+
+			diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
+			assert.Empty(t, diags)
+		})
+	}
+}
+
+func TestValidateSingleNodeClusterPassJobClusters(t *testing.T) {
+	ctx := context.Background()
+
+	for _, tc := range passCases() {
+		t.Run(tc.name, func(t *testing.T) {
+			b := &bundle.Bundle{
+				Config: config.Root{
+					Resources: config.Resources{
+						Jobs: map[string]*resources.Job{
+							"foo": {
+								JobSettings: &jobs.JobSettings{
+									JobClusters: []jobs.JobCluster{
+										{
+											NewCluster: compute.ClusterSpec{
+												ClusterName: "my_cluster",
+												SparkConf:   tc.sparkConf,
+												CustomTags:  tc.customTags,
+												PolicyId:    tc.policyId,
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			}
+
+			if tc.numWorkers != nil {
+				bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
+					return dyn.Set(v, "resources.jobs.foo.job_clusters[0].new_cluster.num_workers", dyn.V(*tc.numWorkers))
+				})
+			}
+
+			diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
+			assert.Empty(t, diags)
+		})
+	}
+}
+
+func TestValidateSingleNodeClusterPassJobTaskClusters(t *testing.T) {
+	ctx := context.Background()
+
+	for _, tc := range passCases() {
+		t.Run(tc.name, func(t *testing.T) {
+			b := &bundle.Bundle{
+				Config: config.Root{
+					Resources: config.Resources{
+						Jobs: map[string]*resources.Job{
+							"foo": {
+								JobSettings: &jobs.JobSettings{
+									Tasks: []jobs.Task{
+										{
+											NewCluster: &compute.ClusterSpec{
+												ClusterName: "my_cluster",
+												SparkConf:   tc.sparkConf,
+												CustomTags:  tc.customTags,
+												PolicyId:    tc.policyId,
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			}
+
+			if tc.numWorkers != nil {
+				bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
+					return dyn.Set(v, "resources.jobs.foo.tasks[0].new_cluster.num_workers", dyn.V(*tc.numWorkers))
+				})
+			}
+
+			diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
+			assert.Empty(t, diags)
+		})
+	}
+}
+
+func TestValidateSingleNodeClusterPassPipelineClusters(t *testing.T) {
+	ctx := context.Background()
+
+	for _, tc := range passCases() {
+		t.Run(tc.name, func(t *testing.T) {
+			b := &bundle.Bundle{
+				Config: config.Root{
+					Resources: config.Resources{
+						Pipelines: map[string]*resources.Pipeline{
+							"foo": {
+								PipelineSpec: &pipelines.PipelineSpec{
+									Clusters: []pipelines.PipelineCluster{
+										{
+											SparkConf:  tc.sparkConf,
+											CustomTags: tc.customTags,
+											PolicyId:   tc.policyId,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			}
+
+			if tc.numWorkers != nil {
+				bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
+					return dyn.Set(v, "resources.pipelines.foo.clusters[0].num_workers", dyn.V(*tc.numWorkers))
+				})
+			}
+
+			diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
+			assert.Empty(t, diags)
+		})
+	}
+}
+
+func TestValidateSingleNodeClusterPassJobForEachTaskCluster(t *testing.T) {
+	ctx := context.Background()
+
+	for _, tc := range passCases() {
+		t.Run(tc.name, func(t *testing.T) {
+			b := &bundle.Bundle{
+				Config: config.Root{
+					Resources: config.Resources{
+						Jobs: map[string]*resources.Job{
+							"foo": {
+								JobSettings: &jobs.JobSettings{
+									Tasks: []jobs.Task{
+										{
+											ForEachTask: &jobs.ForEachTask{
+												Task: jobs.Task{
+													NewCluster: &compute.ClusterSpec{
+														ClusterName: "my_cluster",
+														SparkConf:   tc.sparkConf,
+														CustomTags:  tc.customTags,
+														PolicyId:    tc.policyId,
+													},
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			}
+
+			if tc.numWorkers != nil {
+				bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
+					return dyn.Set(v, "resources.jobs.foo.tasks[0].for_each_task.task.new_cluster.num_workers", dyn.V(*tc.numWorkers))
+				})
+			}
+
+			diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
+			assert.Empty(t, diags)
+		})
+	}
+}

bundle/config/validate/validate.go

@@ -36,6 +36,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 		ValidateSyncPatterns(),
 		JobTaskClusterSpec(),
 		ValidateFolderPermissions(),
+		SingleNodeCluster(),
 	))
 }
 

bundle/config/variable/lookup.go

@@ -1,11 +1,8 @@
-// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT.
-
 package variable
 
 import (
 	"context"
 	"fmt"
-	"strings"
 
 	"github.com/databricks/databricks-sdk-go"
 )
@@ -25,6 +22,8 @@ type Lookup struct {
 
 	Metastore string `json:"metastore,omitempty"`
 
+	NotificationDestination string `json:"notification_destination,omitempty"`
+
 	Pipeline string `json:"pipeline,omitempty"`
 
 	Query string `json:"query,omitempty"`
@@ -34,323 +33,78 @@ type Lookup struct {
 	Warehouse string `json:"warehouse,omitempty"`
 }
 
-func LookupFromMap(m map[string]any) *Lookup {
-	l := &Lookup{}
-	if v, ok := m["alert"]; ok {
-		l.Alert = v.(string)
+type resolver interface {
+	// Resolve resolves the underlying entity's ID.
+	Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error)
+
+	// String returns a human-readable representation of the resolver.
+	String() string
+}
+
+func (l *Lookup) constructResolver() (resolver, error) {
+	var resolvers []resolver
+
+	if l.Alert != "" {
+		resolvers = append(resolvers, resolveAlert{name: l.Alert})
 	}
-	if v, ok := m["cluster_policy"]; ok {
-		l.ClusterPolicy = v.(string)
+	if l.ClusterPolicy != "" {
+		resolvers = append(resolvers, resolveClusterPolicy{name: l.ClusterPolicy})
 	}
-	if v, ok := m["cluster"]; ok {
-		l.Cluster = v.(string)
+	if l.Cluster != "" {
+		resolvers = append(resolvers, resolveCluster{name: l.Cluster})
 	}
-	if v, ok := m["dashboard"]; ok {
-		l.Dashboard = v.(string)
+	if l.Dashboard != "" {
+		resolvers = append(resolvers, resolveDashboard{name: l.Dashboard})
 	}
-	if v, ok := m["instance_pool"]; ok {
-		l.InstancePool = v.(string)
+	if l.InstancePool != "" {
+		resolvers = append(resolvers, resolveInstancePool{name: l.InstancePool})
 	}
-	if v, ok := m["job"]; ok {
-		l.Job = v.(string)
+	if l.Job != "" {
+		resolvers = append(resolvers, resolveJob{name: l.Job})
 	}
-	if v, ok := m["metastore"]; ok {
-		l.Metastore = v.(string)
+	if l.Metastore != "" {
+		resolvers = append(resolvers, resolveMetastore{name: l.Metastore})
 	}
-	if v, ok := m["pipeline"]; ok {
-		l.Pipeline = v.(string)
+	if l.NotificationDestination != "" {
+		resolvers = append(resolvers, resolveNotificationDestination{name: l.NotificationDestination})
 	}
-	if v, ok := m["query"]; ok {
-		l.Query = v.(string)
+	if l.Pipeline != "" {
+		resolvers = append(resolvers, resolvePipeline{name: l.Pipeline})
 	}
-	if v, ok := m["service_principal"]; ok {
-		l.ServicePrincipal = v.(string)
+	if l.Query != "" {
+		resolvers = append(resolvers, resolveQuery{name: l.Query})
 	}
-	if v, ok := m["warehouse"]; ok {
-		l.Warehouse = v.(string)
+	if l.ServicePrincipal != "" {
+		resolvers = append(resolvers, resolveServicePrincipal{name: l.ServicePrincipal})
+	}
+	if l.Warehouse != "" {
+		resolvers = append(resolvers, resolveWarehouse{name: l.Warehouse})
 	}
 
-	return l
+	switch len(resolvers) {
+	case 0:
+		return nil, fmt.Errorf("no valid lookup fields provided")
+	case 1:
+		return resolvers[0], nil
+	default:
+		return nil, fmt.Errorf("exactly one lookup field must be provided")
+	}
 }
 
 func (l *Lookup) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
-	if err := l.validate(); err != nil {
+	r, err := l.constructResolver()
+	if err != nil {
 		return "", err
 	}
 
-	r := allResolvers()
-	if l.Alert != "" {
-		return r.Alert(ctx, w, l.Alert)
-	}
-	if l.ClusterPolicy != "" {
-		return r.ClusterPolicy(ctx, w, l.ClusterPolicy)
-	}
-	if l.Cluster != "" {
-		return r.Cluster(ctx, w, l.Cluster)
-	}
-	if l.Dashboard != "" {
-		return r.Dashboard(ctx, w, l.Dashboard)
-	}
-	if l.InstancePool != "" {
-		return r.InstancePool(ctx, w, l.InstancePool)
-	}
-	if l.Job != "" {
-		return r.Job(ctx, w, l.Job)
-	}
-	if l.Metastore != "" {
-		return r.Metastore(ctx, w, l.Metastore)
-	}
-	if l.Pipeline != "" {
-		return r.Pipeline(ctx, w, l.Pipeline)
-	}
-	if l.Query != "" {
-		return r.Query(ctx, w, l.Query)
-	}
-	if l.ServicePrincipal != "" {
-		return r.ServicePrincipal(ctx, w, l.ServicePrincipal)
-	}
-	if l.Warehouse != "" {
-		return r.Warehouse(ctx, w, l.Warehouse)
-	}
-
-	return "", fmt.Errorf("no valid lookup fields provided")
+	return r.Resolve(ctx, w)
 }
 
 func (l *Lookup) String() string {
-	if l.Alert != "" {
-		return fmt.Sprintf("alert: %s", l.Alert)
-	}
-	if l.ClusterPolicy != "" {
-		return fmt.Sprintf("cluster-policy: %s", l.ClusterPolicy)
-	}
-	if l.Cluster != "" {
-		return fmt.Sprintf("cluster: %s", l.Cluster)
-	}
-	if l.Dashboard != "" {
-		return fmt.Sprintf("dashboard: %s", l.Dashboard)
-	}
-	if l.InstancePool != "" {
-		return fmt.Sprintf("instance-pool: %s", l.InstancePool)
-	}
-	if l.Job != "" {
-		return fmt.Sprintf("job: %s", l.Job)
-	}
-	if l.Metastore != "" {
-		return fmt.Sprintf("metastore: %s", l.Metastore)
-	}
-	if l.Pipeline != "" {
-		return fmt.Sprintf("pipeline: %s", l.Pipeline)
-	}
-	if l.Query != "" {
-		return fmt.Sprintf("query: %s", l.Query)
-	}
-	if l.ServicePrincipal != "" {
-		return fmt.Sprintf("service-principal: %s", l.ServicePrincipal)
-	}
-	if l.Warehouse != "" {
-		return fmt.Sprintf("warehouse: %s", l.Warehouse)
-	}
-
+	r, _ := l.constructResolver()
+	if r == nil {
 		return ""
-}
-
-func (l *Lookup) validate() error {
-	// Validate that only one field is set
-	count := 0
-	if l.Alert != "" {
-		count++
-	}
-	if l.ClusterPolicy != "" {
-		count++
-	}
-	if l.Cluster != "" {
-		count++
-	}
-	if l.Dashboard != "" {
-		count++
-	}
-	if l.InstancePool != "" {
-		count++
-	}
-	if l.Job != "" {
-		count++
-	}
-	if l.Metastore != "" {
-		count++
-	}
-	if l.Pipeline != "" {
-		count++
-	}
-	if l.Query != "" {
-		count++
-	}
-	if l.ServicePrincipal != "" {
-		count++
-	}
-	if l.Warehouse != "" {
-		count++
-	}
-
-	if count != 1 {
-		return fmt.Errorf("exactly one lookup field must be provided")
-	}
-
-	if strings.Contains(l.String(), "${var") {
-		return fmt.Errorf("lookup fields cannot contain variable references")
-	}
-
-	return nil
-}
-
-type resolverFunc func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error)
-type resolvers struct {
-	Alert            resolverFunc
-	ClusterPolicy    resolverFunc
-	Cluster          resolverFunc
-	Dashboard        resolverFunc
-	InstancePool     resolverFunc
-	Job              resolverFunc
-	Metastore        resolverFunc
-	Pipeline         resolverFunc
-	Query            resolverFunc
-	ServicePrincipal resolverFunc
-	Warehouse        resolverFunc
-}
-
-func allResolvers() *resolvers {
-	r := &resolvers{}
-	r.Alert = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["Alert"]
-		if ok {
-			return fn(ctx, w, name)
-		}
-		entity, err := w.Alerts.GetByDisplayName(ctx, name)
-		if err != nil {
-			return "", err
-		}
-
-		return fmt.Sprint(entity.Id), nil
-	}
-	r.ClusterPolicy = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["ClusterPolicy"]
-		if ok {
-			return fn(ctx, w, name)
-		}
-		entity, err := w.ClusterPolicies.GetByName(ctx, name)
-		if err != nil {
-			return "", err
-		}
-
-		return fmt.Sprint(entity.PolicyId), nil
-	}
-	r.Cluster = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["Cluster"]
-		if ok {
-			return fn(ctx, w, name)
-		}
-		entity, err := w.Clusters.GetByClusterName(ctx, name)
-		if err != nil {
-			return "", err
-		}
-
-		return fmt.Sprint(entity.ClusterId), nil
-	}
-	r.Dashboard = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["Dashboard"]
-		if ok {
-			return fn(ctx, w, name)
-		}
-		entity, err := w.Dashboards.GetByName(ctx, name)
-		if err != nil {
-			return "", err
-		}
-
-		return fmt.Sprint(entity.Id), nil
-	}
-	r.InstancePool = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["InstancePool"]
-		if ok {
-			return fn(ctx, w, name)
-		}
-		entity, err := w.InstancePools.GetByInstancePoolName(ctx, name)
-		if err != nil {
-			return "", err
-		}
-
-		return fmt.Sprint(entity.InstancePoolId), nil
-	}
-	r.Job = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["Job"]
-		if ok {
-			return fn(ctx, w, name)
-		}
-		entity, err := w.Jobs.GetBySettingsName(ctx, name)
-		if err != nil {
-			return "", err
-		}
-
-		return fmt.Sprint(entity.JobId), nil
-	}
-	r.Metastore = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["Metastore"]
-		if ok {
-			return fn(ctx, w, name)
-		}
-		entity, err := w.Metastores.GetByName(ctx, name)
-		if err != nil {
-			return "", err
-		}
-
-		return fmt.Sprint(entity.MetastoreId), nil
-	}
-	r.Pipeline = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["Pipeline"]
-		if ok {
-			return fn(ctx, w, name)
-		}
-		entity, err := w.Pipelines.GetByName(ctx, name)
-		if err != nil {
-			return "", err
-		}
-
-		return fmt.Sprint(entity.PipelineId), nil
-	}
-	r.Query = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["Query"]
-		if ok {
-			return fn(ctx, w, name)
-		}
-		entity, err := w.Queries.GetByDisplayName(ctx, name)
-		if err != nil {
-			return "", err
-		}
-
-		return fmt.Sprint(entity.Id), nil
-	}
-	r.ServicePrincipal = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["ServicePrincipal"]
-		if ok {
-			return fn(ctx, w, name)
-		}
-		entity, err := w.ServicePrincipals.GetByDisplayName(ctx, name)
-		if err != nil {
-			return "", err
-		}
-
-		return fmt.Sprint(entity.ApplicationId), nil
-	}
-	r.Warehouse = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["Warehouse"]
-		if ok {
-			return fn(ctx, w, name)
-		}
-		entity, err := w.Warehouses.GetByName(ctx, name)
-		if err != nil {
-			return "", err
-		}
-
-		return fmt.Sprint(entity.Id), nil
-	}
-
-	return r
+	}
+
+	return r.String()
 }

bundle/config/variable/lookup_test.go

@@ -0,0 +1,60 @@
+package variable
+
+import (
+	"context"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLookup_Coverage(t *testing.T) {
+	var lookup Lookup
+	val := reflect.ValueOf(lookup)
+	typ := val.Type()
+
+	for i := 0; i < val.NumField(); i++ {
+		field := val.Field(i)
+		if field.Kind() != reflect.String {
+			t.Fatalf("Field %s is not a string", typ.Field(i).Name)
+		}
+
+		fieldType := typ.Field(i)
+		t.Run(fieldType.Name, func(t *testing.T) {
+			// Use a fresh instance of the struct in each test
+			var lookup Lookup
+
+			// Set the field to a non-empty string
+			reflect.ValueOf(&lookup).Elem().Field(i).SetString("value")
+
+			// Test the [String] function
+			assert.NotEmpty(t, lookup.String())
+		})
+	}
+}
+
+func TestLookup_Empty(t *testing.T) {
+	var lookup Lookup
+
+	// Resolve returns an error when no fields are provided
+	_, err := lookup.Resolve(context.Background(), nil)
+	assert.ErrorContains(t, err, "no valid lookup fields provided")
+
+	// No string representation for an invalid lookup
+	assert.Empty(t, lookup.String())
+
+}
+
+func TestLookup_Multiple(t *testing.T) {
+	lookup := Lookup{
+		Alert: "alert",
+		Query: "query",
+	}
+
+	// Resolve returns an error when multiple fields are provided
+	_, err := lookup.Resolve(context.Background(), nil)
+	assert.ErrorContains(t, err, "exactly one lookup field must be provided")
+
+	// No string representation for an invalid lookup
+	assert.Empty(t, lookup.String())
+}

bundle/config/variable/resolve_alert.go

@@ -0,0 +1,24 @@
+package variable
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/databricks-sdk-go"
+)
+
+type resolveAlert struct {
+	name string
+}
+
+func (l resolveAlert) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
+	entity, err := w.Alerts.GetByDisplayName(ctx, l.name)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprint(entity.Id), nil
+}
+
+func (l resolveAlert) String() string {
+	return fmt.Sprintf("alert: %s", l.name)
+}

bundle/config/variable/resolve_alert_test.go

@@ -0,0 +1,49 @@
+package variable
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/apierr"
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/service/sql"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolveAlert_ResolveSuccess(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockAlertsAPI()
+	api.EXPECT().
+		GetByDisplayName(mock.Anything, "alert").
+		Return(&sql.ListAlertsResponseAlert{
+			Id: "1234",
+		}, nil)
+
+	ctx := context.Background()
+	l := resolveAlert{name: "alert"}
+	result, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.NoError(t, err)
+	assert.Equal(t, "1234", result)
+}
+
+func TestResolveAlert_ResolveNotFound(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockAlertsAPI()
+	api.EXPECT().
+		GetByDisplayName(mock.Anything, "alert").
+		Return(nil, &apierr.APIError{StatusCode: 404})
+
+	ctx := context.Background()
+	l := resolveAlert{name: "alert"}
+	_, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.ErrorIs(t, err, apierr.ErrNotFound)
+}
+
+func TestResolveAlert_String(t *testing.T) {
+	l := resolveAlert{name: "name"}
+	assert.Equal(t, "alert: name", l.String())
+}

bundle/config/variable/resolve_cluster.go

@@ -8,13 +8,13 @@ import (
 	"github.com/databricks/databricks-sdk-go/service/compute"
 )
 
-var lookupOverrides = map[string]resolverFunc{
-	"Cluster": resolveCluster,
+type resolveCluster struct {
+	name string
 }
 
 // We added a custom resolver for the cluster to add filtering for the cluster source when we list all clusters.
 // Without the filtering listing could take a very long time (5-10 mins) which leads to lookup timeouts.
-func resolveCluster(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
+func (l resolveCluster) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
 	result, err := w.Clusters.ListAll(ctx, compute.ListClustersRequest{
 		FilterBy: &compute.ListClustersFilterBy{
 			ClusterSources: []compute.ClusterSource{compute.ClusterSourceApi, compute.ClusterSourceUi},
@@ -30,6 +30,8 @@ func resolveCluster(ctx context.Context, w *databricks.WorkspaceClient, name str
 		key := v.ClusterName
 		tmp[key] = append(tmp[key], v)
 	}
+
+	name := l.name
 	alternatives, ok := tmp[name]
 	if !ok || len(alternatives) == 0 {
 		return "", fmt.Errorf("cluster named '%s' does not exist", name)
@@ -39,3 +41,7 @@ func resolveCluster(ctx context.Context, w *databricks.WorkspaceClient, name str
 	}
 	return alternatives[0].ClusterId, nil
 }
+
+func (l resolveCluster) String() string {
+	return fmt.Sprintf("cluster: %s", l.name)
+}

bundle/config/variable/resolve_cluster_policy.go

@@ -0,0 +1,24 @@
+package variable
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/databricks-sdk-go"
+)
+
+type resolveClusterPolicy struct {
+	name string
+}
+
+func (l resolveClusterPolicy) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
+	entity, err := w.ClusterPolicies.GetByName(ctx, l.name)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprint(entity.PolicyId), nil
+}
+
+func (l resolveClusterPolicy) String() string {
+	return fmt.Sprintf("cluster-policy: %s", l.name)
+}

bundle/config/variable/resolve_cluster_policy_test.go

@@ -0,0 +1,49 @@
+package variable
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/apierr"
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/service/compute"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolveClusterPolicy_ResolveSuccess(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockClusterPoliciesAPI()
+	api.EXPECT().
+		GetByName(mock.Anything, "policy").
+		Return(&compute.Policy{
+			PolicyId: "1234",
+		}, nil)
+
+	ctx := context.Background()
+	l := resolveClusterPolicy{name: "policy"}
+	result, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.NoError(t, err)
+	assert.Equal(t, "1234", result)
+}
+
+func TestResolveClusterPolicy_ResolveNotFound(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockClusterPoliciesAPI()
+	api.EXPECT().
+		GetByName(mock.Anything, "policy").
+		Return(nil, &apierr.APIError{StatusCode: 404})
+
+	ctx := context.Background()
+	l := resolveClusterPolicy{name: "policy"}
+	_, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.ErrorIs(t, err, apierr.ErrNotFound)
+}
+
+func TestResolveClusterPolicy_String(t *testing.T) {
+	l := resolveClusterPolicy{name: "name"}
+	assert.Equal(t, "cluster-policy: name", l.String())
+}

bundle/config/variable/resolve_cluster_test.go

@@ -0,0 +1,50 @@
+package variable
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/service/compute"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolveCluster_ResolveSuccess(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockClustersAPI()
+	api.EXPECT().
+		ListAll(mock.Anything, mock.Anything).
+		Return([]compute.ClusterDetails{
+			{ClusterId: "1234", ClusterName: "cluster1"},
+			{ClusterId: "2345", ClusterName: "cluster2"},
+		}, nil)
+
+	ctx := context.Background()
+	l := resolveCluster{name: "cluster2"}
+	result, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.NoError(t, err)
+	assert.Equal(t, "2345", result)
+}
+
+func TestResolveCluster_ResolveNotFound(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockClustersAPI()
+	api.EXPECT().
+		ListAll(mock.Anything, mock.Anything).
+		Return([]compute.ClusterDetails{}, nil)
+
+	ctx := context.Background()
+	l := resolveCluster{name: "cluster"}
+	_, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "cluster named 'cluster' does not exist")
+}
+
+func TestResolveCluster_String(t *testing.T) {
+	l := resolveCluster{name: "name"}
+	assert.Equal(t, "cluster: name", l.String())
+}

bundle/config/variable/resolve_dashboard.go

@@ -0,0 +1,24 @@
+package variable
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/databricks-sdk-go"
+)
+
+type resolveDashboard struct {
+	name string
+}
+
+func (l resolveDashboard) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
+	entity, err := w.Dashboards.GetByName(ctx, l.name)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprint(entity.Id), nil
+}
+
+func (l resolveDashboard) String() string {
+	return fmt.Sprintf("dashboard: %s", l.name)
+}

bundle/config/variable/resolve_dashboard_test.go

@@ -0,0 +1,49 @@
+package variable
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/apierr"
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/service/sql"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolveDashboard_ResolveSuccess(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockDashboardsAPI()
+	api.EXPECT().
+		GetByName(mock.Anything, "dashboard").
+		Return(&sql.Dashboard{
+			Id: "1234",
+		}, nil)
+
+	ctx := context.Background()
+	l := resolveDashboard{name: "dashboard"}
+	result, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.NoError(t, err)
+	assert.Equal(t, "1234", result)
+}
+
+func TestResolveDashboard_ResolveNotFound(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockDashboardsAPI()
+	api.EXPECT().
+		GetByName(mock.Anything, "dashboard").
+		Return(nil, &apierr.APIError{StatusCode: 404})
+
+	ctx := context.Background()
+	l := resolveDashboard{name: "dashboard"}
+	_, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.ErrorIs(t, err, apierr.ErrNotFound)
+}
+
+func TestResolveDashboard_String(t *testing.T) {
+	l := resolveDashboard{name: "name"}
+	assert.Equal(t, "dashboard: name", l.String())
+}

bundle/config/variable/resolve_instance_pool.go

@@ -0,0 +1,24 @@
+package variable
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/databricks-sdk-go"
+)
+
+type resolveInstancePool struct {
+	name string
+}
+
+func (l resolveInstancePool) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
+	entity, err := w.InstancePools.GetByInstancePoolName(ctx, l.name)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprint(entity.InstancePoolId), nil
+}
+
+func (l resolveInstancePool) String() string {
+	return fmt.Sprintf("instance-pool: %s", l.name)
+}

bundle/config/variable/resolve_instance_pool_test.go

@@ -0,0 +1,49 @@
+package variable
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/apierr"
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/service/compute"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolveInstancePool_ResolveSuccess(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockInstancePoolsAPI()
+	api.EXPECT().
+		GetByInstancePoolName(mock.Anything, "instance_pool").
+		Return(&compute.InstancePoolAndStats{
+			InstancePoolId: "5678",
+		}, nil)
+
+	ctx := context.Background()
+	l := resolveInstancePool{name: "instance_pool"}
+	result, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.NoError(t, err)
+	assert.Equal(t, "5678", result)
+}
+
+func TestResolveInstancePool_ResolveNotFound(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockInstancePoolsAPI()
+	api.EXPECT().
+		GetByInstancePoolName(mock.Anything, "instance_pool").
+		Return(nil, &apierr.APIError{StatusCode: 404})
+
+	ctx := context.Background()
+	l := resolveInstancePool{name: "instance_pool"}
+	_, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.ErrorIs(t, err, apierr.ErrNotFound)
+}
+
+func TestResolveInstancePool_String(t *testing.T) {
+	l := resolveInstancePool{name: "name"}
+	assert.Equal(t, "instance-pool: name", l.String())
+}

bundle/config/variable/resolve_job.go

@@ -0,0 +1,24 @@
+package variable
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/databricks-sdk-go"
+)
+
+type resolveJob struct {
+	name string
+}
+
+func (l resolveJob) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
+	entity, err := w.Jobs.GetBySettingsName(ctx, l.name)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprint(entity.JobId), nil
+}
+
+func (l resolveJob) String() string {
+	return fmt.Sprintf("job: %s", l.name)
+}

bundle/config/variable/resolve_job_test.go

@@ -0,0 +1,49 @@
+package variable
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/apierr"
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolveJob_ResolveSuccess(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockJobsAPI()
+	api.EXPECT().
+		GetBySettingsName(mock.Anything, "job").
+		Return(&jobs.BaseJob{
+			JobId: 5678,
+		}, nil)
+
+	ctx := context.Background()
+	l := resolveJob{name: "job"}
+	result, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.NoError(t, err)
+	assert.Equal(t, "5678", result)
+}
+
+func TestResolveJob_ResolveNotFound(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockJobsAPI()
+	api.EXPECT().
+		GetBySettingsName(mock.Anything, "job").
+		Return(nil, &apierr.APIError{StatusCode: 404})
+
+	ctx := context.Background()
+	l := resolveJob{name: "job"}
+	_, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.ErrorIs(t, err, apierr.ErrNotFound)
+}
+
+func TestResolveJob_String(t *testing.T) {
+	l := resolveJob{name: "name"}
+	assert.Equal(t, "job: name", l.String())
+}

bundle/config/variable/resolve_metastore.go

@@ -0,0 +1,24 @@
+package variable
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/databricks-sdk-go"
+)
+
+type resolveMetastore struct {
+	name string
+}
+
+func (l resolveMetastore) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
+	entity, err := w.Metastores.GetByName(ctx, l.name)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprint(entity.MetastoreId), nil
+}
+
+func (l resolveMetastore) String() string {
+	return fmt.Sprintf("metastore: %s", l.name)
+}

bundle/config/variable/resolve_metastore_test.go

@@ -0,0 +1,49 @@
+package variable
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/apierr"
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/service/catalog"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolveMetastore_ResolveSuccess(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockMetastoresAPI()
+	api.EXPECT().
+		GetByName(mock.Anything, "metastore").
+		Return(&catalog.MetastoreInfo{
+			MetastoreId: "abcd",
+		}, nil)
+
+	ctx := context.Background()
+	l := resolveMetastore{name: "metastore"}
+	result, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.NoError(t, err)
+	assert.Equal(t, "abcd", result)
+}
+
+func TestResolveMetastore_ResolveNotFound(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockMetastoresAPI()
+	api.EXPECT().
+		GetByName(mock.Anything, "metastore").
+		Return(nil, &apierr.APIError{StatusCode: 404})
+
+	ctx := context.Background()
+	l := resolveMetastore{name: "metastore"}
+	_, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.ErrorIs(t, err, apierr.ErrNotFound)
+}
+
+func TestResolveMetastore_String(t *testing.T) {
+	l := resolveMetastore{name: "name"}
+	assert.Equal(t, "metastore: name", l.String())
+}

bundle/config/variable/resolve_notification_destination.go

@@ -0,0 +1,46 @@
+package variable
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/service/settings"
+)
+
+type resolveNotificationDestination struct {
+	name string
+}
+
+func (l resolveNotificationDestination) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
+	result, err := w.NotificationDestinations.ListAll(ctx, settings.ListNotificationDestinationsRequest{
+		// The default page size for this API is 20.
+		// We use a higher value to make fewer API calls.
+		PageSize: 200,
+	})
+	if err != nil {
+		return "", err
+	}
+
+	// Collect all notification destinations with the given name.
+	var entities []settings.ListNotificationDestinationsResult
+	for _, entity := range result {
+		if entity.DisplayName == l.name {
+			entities = append(entities, entity)
+		}
+	}
+
+	// Return the ID of the first matching notification destination.
+	switch len(entities) {
+	case 0:
+		return "", fmt.Errorf("notification destination named %q does not exist", l.name)
+	case 1:
+		return entities[0].Id, nil
+	default:
+		return "", fmt.Errorf("there are %d instances of clusters named %q", len(entities), l.name)
+	}
+}
+
+func (l resolveNotificationDestination) String() string {
+	return fmt.Sprintf("notification-destination: %s", l.name)
+}

bundle/config/variable/resolve_notification_destination_test.go

@@ -0,0 +1,82 @@
+package variable
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/service/settings"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolveNotificationDestination_ResolveSuccess(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockNotificationDestinationsAPI()
+	api.EXPECT().
+		ListAll(mock.Anything, mock.Anything).
+		Return([]settings.ListNotificationDestinationsResult{
+			{Id: "1234", DisplayName: "destination"},
+		}, nil)
+
+	ctx := context.Background()
+	l := resolveNotificationDestination{name: "destination"}
+	result, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.NoError(t, err)
+	assert.Equal(t, "1234", result)
+}
+
+func TestResolveNotificationDestination_ResolveError(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockNotificationDestinationsAPI()
+	api.EXPECT().
+		ListAll(mock.Anything, mock.Anything).
+		Return(nil, fmt.Errorf("bad"))
+
+	ctx := context.Background()
+	l := resolveNotificationDestination{name: "destination"}
+	_, err := l.Resolve(ctx, m.WorkspaceClient)
+	assert.ErrorContains(t, err, "bad")
+}
+
+func TestResolveNotificationDestination_ResolveNotFound(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockNotificationDestinationsAPI()
+	api.EXPECT().
+		ListAll(mock.Anything, mock.Anything).
+		Return([]settings.ListNotificationDestinationsResult{}, nil)
+
+	ctx := context.Background()
+	l := resolveNotificationDestination{name: "destination"}
+	_, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.Error(t, err)
+	assert.ErrorContains(t, err, `notification destination named "destination" does not exist`)
+}
+
+func TestResolveNotificationDestination_ResolveMultiple(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockNotificationDestinationsAPI()
+	api.EXPECT().
+		ListAll(mock.Anything, mock.Anything).
+		Return([]settings.ListNotificationDestinationsResult{
+			{Id: "1234", DisplayName: "destination"},
+			{Id: "5678", DisplayName: "destination"},
+		}, nil)
+
+	ctx := context.Background()
+	l := resolveNotificationDestination{name: "destination"}
+	_, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.Error(t, err)
+	assert.ErrorContains(t, err, `there are 2 instances of clusters named "destination"`)
+}
+
+func TestResolveNotificationDestination_String(t *testing.T) {
+	l := resolveNotificationDestination{name: "name"}
+	assert.Equal(t, "notification-destination: name", l.String())
+}

bundle/config/variable/resolve_pipeline.go

@@ -0,0 +1,24 @@
+package variable
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/databricks-sdk-go"
+)
+
+type resolvePipeline struct {
+	name string
+}
+
+func (l resolvePipeline) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
+	entity, err := w.Pipelines.GetByName(ctx, l.name)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprint(entity.PipelineId), nil
+}
+
+func (l resolvePipeline) String() string {
+	return fmt.Sprintf("pipeline: %s", l.name)
+}

bundle/config/variable/resolve_pipeline_test.go

@@ -0,0 +1,49 @@
+package variable
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/apierr"
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/service/pipelines"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolvePipeline_ResolveSuccess(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockPipelinesAPI()
+	api.EXPECT().
+		GetByName(mock.Anything, "pipeline").
+		Return(&pipelines.PipelineStateInfo{
+			PipelineId: "abcd",
+		}, nil)
+
+	ctx := context.Background()
+	l := resolvePipeline{name: "pipeline"}
+	result, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.NoError(t, err)
+	assert.Equal(t, "abcd", result)
+}
+
+func TestResolvePipeline_ResolveNotFound(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockPipelinesAPI()
+	api.EXPECT().
+		GetByName(mock.Anything, "pipeline").
+		Return(nil, &apierr.APIError{StatusCode: 404})
+
+	ctx := context.Background()
+	l := resolvePipeline{name: "pipeline"}
+	_, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.ErrorIs(t, err, apierr.ErrNotFound)
+}
+
+func TestResolvePipeline_String(t *testing.T) {
+	l := resolvePipeline{name: "name"}
+	assert.Equal(t, "pipeline: name", l.String())
+}

bundle/config/variable/resolve_query.go

@@ -0,0 +1,24 @@
+package variable
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/databricks-sdk-go"
+)
+
+type resolveQuery struct {
+	name string
+}
+
+func (l resolveQuery) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
+	entity, err := w.Queries.GetByDisplayName(ctx, l.name)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprint(entity.Id), nil
+}
+
+func (l resolveQuery) String() string {
+	return fmt.Sprintf("query: %s", l.name)
+}

bundle/config/variable/resolve_query_test.go

@@ -0,0 +1,49 @@
+package variable
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/apierr"
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/service/sql"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolveQuery_ResolveSuccess(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockQueriesAPI()
+	api.EXPECT().
+		GetByDisplayName(mock.Anything, "query").
+		Return(&sql.ListQueryObjectsResponseQuery{
+			Id: "1234",
+		}, nil)
+
+	ctx := context.Background()
+	l := resolveQuery{name: "query"}
+	result, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.NoError(t, err)
+	assert.Equal(t, "1234", result)
+}
+
+func TestResolveQuery_ResolveNotFound(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockQueriesAPI()
+	api.EXPECT().
+		GetByDisplayName(mock.Anything, "query").
+		Return(nil, &apierr.APIError{StatusCode: 404})
+
+	ctx := context.Background()
+	l := resolveQuery{name: "query"}
+	_, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.ErrorIs(t, err, apierr.ErrNotFound)
+}
+
+func TestResolveQuery_String(t *testing.T) {
+	l := resolveQuery{name: "name"}
+	assert.Equal(t, "query: name", l.String())
+}

bundle/config/variable/resolve_service_principal.go

@@ -0,0 +1,24 @@
+package variable
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/databricks-sdk-go"
+)
+
+type resolveServicePrincipal struct {
+	name string
+}
+
+func (l resolveServicePrincipal) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
+	entity, err := w.ServicePrincipals.GetByDisplayName(ctx, l.name)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprint(entity.ApplicationId), nil
+}
+
+func (l resolveServicePrincipal) String() string {
+	return fmt.Sprintf("service-principal: %s", l.name)
+}

bundle/config/variable/resolve_service_principal_test.go

@@ -0,0 +1,49 @@
+package variable
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/apierr"
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/service/iam"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolveServicePrincipal_ResolveSuccess(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockServicePrincipalsAPI()
+	api.EXPECT().
+		GetByDisplayName(mock.Anything, "service-principal").
+		Return(&iam.ServicePrincipal{
+			ApplicationId: "5678",
+		}, nil)
+
+	ctx := context.Background()
+	l := resolveServicePrincipal{name: "service-principal"}
+	result, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.NoError(t, err)
+	assert.Equal(t, "5678", result)
+}
+
+func TestResolveServicePrincipal_ResolveNotFound(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockServicePrincipalsAPI()
+	api.EXPECT().
+		GetByDisplayName(mock.Anything, "service-principal").
+		Return(nil, &apierr.APIError{StatusCode: 404})
+
+	ctx := context.Background()
+	l := resolveServicePrincipal{name: "service-principal"}
+	_, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.ErrorIs(t, err, apierr.ErrNotFound)
+}
+
+func TestResolveServicePrincipal_String(t *testing.T) {
+	l := resolveServicePrincipal{name: "name"}
+	assert.Equal(t, "service-principal: name", l.String())
+}

bundle/config/variable/resolve_warehouse.go

@@ -0,0 +1,24 @@
+package variable
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/databricks-sdk-go"
+)
+
+type resolveWarehouse struct {
+	name string
+}
+
+func (l resolveWarehouse) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
+	entity, err := w.Warehouses.GetByName(ctx, l.name)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprint(entity.Id), nil
+}
+
+func (l resolveWarehouse) String() string {
+	return fmt.Sprintf("warehouse: %s", l.name)
+}

bundle/config/variable/resolve_warehouse_test.go

@@ -0,0 +1,49 @@
+package variable
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/apierr"
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/service/sql"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolveWarehouse_ResolveSuccess(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockWarehousesAPI()
+	api.EXPECT().
+		GetByName(mock.Anything, "warehouse").
+		Return(&sql.EndpointInfo{
+			Id: "abcd",
+		}, nil)
+
+	ctx := context.Background()
+	l := resolveWarehouse{name: "warehouse"}
+	result, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.NoError(t, err)
+	assert.Equal(t, "abcd", result)
+}
+
+func TestResolveWarehouse_ResolveNotFound(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockWarehousesAPI()
+	api.EXPECT().
+		GetByName(mock.Anything, "warehouse").
+		Return(nil, &apierr.APIError{StatusCode: 404})
+
+	ctx := context.Background()
+	l := resolveWarehouse{name: "warehouse"}
+	_, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.ErrorIs(t, err, apierr.ErrNotFound)
+}
+
+func TestResolveWarehouse_String(t *testing.T) {
+	l := resolveWarehouse{name: "name"}
+	assert.Equal(t, "warehouse: name", l.String())
+}

cmd/bundle/generate/generate_test.go

@@ -3,8 +3,10 @@ package generate
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"testing"
@@ -90,7 +92,7 @@ func TestGeneratePipelineCommand(t *testing.T) {
 	err := cmd.RunE(cmd, []string{})
 	require.NoError(t, err)
 
-	data, err := os.ReadFile(filepath.Join(configDir, "test_pipeline.yml"))
+	data, err := os.ReadFile(filepath.Join(configDir, "test_pipeline.pipeline.yml"))
 	require.NoError(t, err)
 	require.Equal(t, fmt.Sprintf(`resources:
   pipelines:
@@ -186,7 +188,123 @@ func TestGenerateJobCommand(t *testing.T) {
 	err := cmd.RunE(cmd, []string{})
 	require.NoError(t, err)
 
-	data, err := os.ReadFile(filepath.Join(configDir, "test_job.yml"))
+	data, err := os.ReadFile(filepath.Join(configDir, "test_job.job.yml"))
+	require.NoError(t, err)
+
+	require.Equal(t, fmt.Sprintf(`resources:
+  jobs:
+    test_job:
+      name: test-job
+      job_clusters:
+        - new_cluster:
+            custom_tags:
+              "Tag1": "24X7-1234"
+        - new_cluster:
+            spark_conf:
+              "spark.databricks.delta.preview.enabled": "true"
+      tasks:
+        - task_key: notebook_task
+          notebook_task:
+            notebook_path: %s
+      parameters:
+        - name: empty
+          default: ""
+`, filepath.Join("..", "src", "notebook.py")), string(data))
+
+	data, err = os.ReadFile(filepath.Join(srcDir, "notebook.py"))
+	require.NoError(t, err)
+	require.Equal(t, "# Databricks notebook source\nNotebook content", string(data))
+}
+
+func touchEmptyFile(t *testing.T, path string) {
+	err := os.MkdirAll(filepath.Dir(path), 0700)
+	require.NoError(t, err)
+	f, err := os.Create(path)
+	require.NoError(t, err)
+	f.Close()
+}
+
+func TestGenerateJobCommandOldFileRename(t *testing.T) {
+	cmd := NewGenerateJobCommand()
+
+	root := t.TempDir()
+	b := &bundle.Bundle{
+		BundleRootPath: root,
+	}
+
+	m := mocks.NewMockWorkspaceClient(t)
+	b.SetWorkpaceClient(m.WorkspaceClient)
+
+	jobsApi := m.GetMockJobsAPI()
+	jobsApi.EXPECT().Get(mock.Anything, jobs.GetJobRequest{JobId: 1234}).Return(&jobs.Job{
+		Settings: &jobs.JobSettings{
+			Name: "test-job",
+			JobClusters: []jobs.JobCluster{
+				{NewCluster: compute.ClusterSpec{
+					CustomTags: map[string]string{
+						"Tag1": "24X7-1234",
+					},
+				}},
+				{NewCluster: compute.ClusterSpec{
+					SparkConf: map[string]string{
+						"spark.databricks.delta.preview.enabled": "true",
+					},
+				}},
+			},
+			Tasks: []jobs.Task{
+				{
+					TaskKey: "notebook_task",
+					NotebookTask: &jobs.NotebookTask{
+						NotebookPath: "/test/notebook",
+					},
+				},
+			},
+			Parameters: []jobs.JobParameterDefinition{
+				{
+					Name:    "empty",
+					Default: "",
+				},
+			},
+		},
+	}, nil)
+
+	workspaceApi := m.GetMockWorkspaceAPI()
+	workspaceApi.EXPECT().GetStatusByPath(mock.Anything, "/test/notebook").Return(&workspace.ObjectInfo{
+		ObjectType: workspace.ObjectTypeNotebook,
+		Language:   workspace.LanguagePython,
+		Path:       "/test/notebook",
+	}, nil)
+
+	notebookContent := io.NopCloser(bytes.NewBufferString("# Databricks notebook source\nNotebook content"))
+	workspaceApi.EXPECT().Download(mock.Anything, "/test/notebook", mock.Anything).Return(notebookContent, nil)
+
+	cmd.SetContext(bundle.Context(context.Background(), b))
+	cmd.Flag("existing-job-id").Value.Set("1234")
+
+	configDir := filepath.Join(root, "resources")
+	cmd.Flag("config-dir").Value.Set(configDir)
+
+	srcDir := filepath.Join(root, "src")
+	cmd.Flag("source-dir").Value.Set(srcDir)
+
+	var key string
+	cmd.Flags().StringVar(&key, "key", "test_job", "")
+
+	// Create an old generated file first
+	oldFilename := filepath.Join(configDir, "test_job.yml")
+	touchEmptyFile(t, oldFilename)
+
+	// Having an existing files require --force flag to regenerate them
+	cmd.Flag("force").Value.Set("true")
+
+	err := cmd.RunE(cmd, []string{})
+	require.NoError(t, err)
+
+	// Make sure file do not exists after the run
+	_, err = os.Stat(oldFilename)
+	require.True(t, errors.Is(err, fs.ErrNotExist))
+
+	data, err := os.ReadFile(filepath.Join(configDir, "test_job.job.yml"))
 	require.NoError(t, err)
 
 	require.Equal(t, fmt.Sprintf(`resources:

cmd/bundle/generate/job.go

@@ -1,7 +1,9 @@
 package generate
 
 import (
+	"errors"
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 
@@ -83,7 +85,17 @@ func NewGenerateJobCommand() *cobra.Command {
 			return err
 		}
 
-		filename := filepath.Join(configDir, fmt.Sprintf("%s.yml", jobKey))
+		oldFilename := filepath.Join(configDir, fmt.Sprintf("%s.yml", jobKey))
+		filename := filepath.Join(configDir, fmt.Sprintf("%s.job.yml", jobKey))
+
+		// User might continuously run generate command to update their bundle jobs with any changes made in Databricks UI.
+		// Due to changing in the generated file names, we need to first rename existing resource file to the new name.
+		// Otherwise users can end up with duplicated resources.
+		err = os.Rename(oldFilename, filename)
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			return fmt.Errorf("failed to rename file %s. DABs uses the resource type as a sub-extension for generated content, please rename it to %s, err: %w", oldFilename, filename, err)
+		}
+
 		saver := yamlsaver.NewSaverWithStyle(map[string]yaml.Style{
 			// Including all JobSettings and nested fields which are map[string]string type
 			"spark_conf":  yaml.DoubleQuotedStyle,

cmd/bundle/generate/pipeline.go

@@ -1,7 +1,9 @@
 package generate
 
 import (
+	"errors"
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 
@@ -83,7 +85,17 @@ func NewGeneratePipelineCommand() *cobra.Command {
 			return err
 		}
 
-		filename := filepath.Join(configDir, fmt.Sprintf("%s.yml", pipelineKey))
+		oldFilename := filepath.Join(configDir, fmt.Sprintf("%s.yml", pipelineKey))
+		filename := filepath.Join(configDir, fmt.Sprintf("%s.pipeline.yml", pipelineKey))
+
+		// User might continuously run generate command to update their bundle jobs with any changes made in Databricks UI.
+		// Due to changing in the generated file names, we need to first rename existing resource file to the new name.
+		// Otherwise users can end up with duplicated resources.
+		err = os.Rename(oldFilename, filename)
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			return fmt.Errorf("failed to rename file %s. DABs uses the resource type as a sub-extension for generated content, please rename it to %s, err: %w", oldFilename, filename, err)
+		}
+
 		saver := yamlsaver.NewSaverWithStyle(
 			// Including all PipelineSpec and nested fields which are map[string]string type
 			map[string]yaml.Style{

go.mod

@@ -27,7 +27,7 @@ require (
 	golang.org/x/mod v0.22.0
 	golang.org/x/oauth2 v0.24.0
 	golang.org/x/sync v0.9.0
-	golang.org/x/term v0.25.0
+	golang.org/x/term v0.26.0
 	golang.org/x/text v0.20.0
 	gopkg.in/ini.v1 v1.67.0 // Apache 2.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -64,7 +64,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	golang.org/x/crypto v0.24.0 // indirect
 	golang.org/x/net v0.26.0 // indirect
-	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/sys v0.27.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	google.golang.org/api v0.182.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect

go.sum

@@ -212,10 +212,10 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
-golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
-golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
+golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
+golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU=
+golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=

internal/bundle/bind_resource_test.go

@@ -166,7 +166,7 @@ func TestAccGenerateAndBind(t *testing.T) {
 	_, err = os.Stat(filepath.Join(bundleRoot, "src", "test.py"))
 	require.NoError(t, err)
 
-	matches, err := filepath.Glob(filepath.Join(bundleRoot, "resources", "test_job_key.yml"))
+	matches, err := filepath.Glob(filepath.Join(bundleRoot, "resources", "test_job_key.job.yml"))
 	require.NoError(t, err)
 
 	require.Len(t, matches, 1)

internal/bundle/bundles/basic/databricks_template_schema.json

@@ -11,6 +11,11 @@
         "node_type_id": {
             "type": "string",
             "description": "Node type id for job cluster"
+        },
+        "root_path": {
+            "type": "string",
+            "description": "Root path to deploy bundle to",
+            "default": ""
         }
     }
 }

internal/bundle/bundles/basic/template/databricks.yml.tmpl

@@ -2,7 +2,11 @@ bundle:
   name: basic
 
 workspace:
+  {{ if .root_path }}
+  root_path: "{{.root_path}}/.bundle/{{.unique_id}}"
+  {{ else }}
   root_path: "~/.bundle/{{.unique_id}}"
+  {{ end }}
 
 resources:
   jobs:

internal/bundle/bundles/empty_bundle/databricks.yml

@@ -1,2 +0,0 @@
-bundle:
-  name: abc

internal/bundle/bundles/recreate_pipeline/template/databricks.yml.tmpl

@@ -1,5 +1,8 @@
 bundle:
-  name: "bundle-playground"
+  name: recreate-pipeline
+
+workspace:
+  root_path: "~/.bundle/{{.unique_id}}"
 
 variables:
   catalog:

internal/bundle/bundles/uc_schema/template/databricks.yml.tmpl

@@ -1,5 +1,8 @@
 bundle:
-  name: "bundle-playground"
+  name: uc-schema
+
+workspace:
+  root_path: "~/.bundle/{{.unique_id}}"
 
 resources:
   pipelines:

internal/bundle/deploy_to_shared_test.go

@@ -0,0 +1,38 @@
+package bundle
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/databricks/cli/internal"
+	"github.com/databricks/cli/internal/acc"
+	"github.com/databricks/cli/libs/env"
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAccDeployBasicToSharedWorkspacePath(t *testing.T) {
+	ctx, wt := acc.WorkspaceTest(t)
+
+	nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV"))
+	uniqueId := uuid.New().String()
+
+	currentUser, err := wt.W.CurrentUser.Me(ctx)
+	require.NoError(t, err)
+
+	bundleRoot, err := initTestTemplate(t, ctx, "basic", map[string]any{
+		"unique_id":     uniqueId,
+		"node_type_id":  nodeTypeId,
+		"spark_version": defaultSparkVersion,
+		"root_path":     fmt.Sprintf("/Shared/%s", currentUser.UserName),
+	})
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		err = destroyBundle(wt.T, ctx, bundleRoot)
+		require.NoError(wt.T, err)
+	})
+
+	err = deployBundle(wt.T, ctx, bundleRoot)
+	require.NoError(wt.T, err)
+}

internal/init_test.go

@@ -97,7 +97,7 @@ func TestAccBundleInitOnMlopsStacks(t *testing.T) {
 	require.NoError(t, err)
 	job, err := w.Jobs.GetByJobId(context.Background(), batchJobId)
 	assert.NoError(t, err)
-	assert.Equal(t, fmt.Sprintf("dev-%s-batch-inference-job", projectName), job.Settings.Name)
+	assert.Contains(t, job.Settings.Name, fmt.Sprintf("dev-%s-batch-inference-job", projectName))
 }
 
 func TestAccBundleInitHelpers(t *testing.T) {