.codegen.json

@@ -11,10 +11,10 @@
     "toolchain": {
         "required": ["go"],
         "post_generate": [
-            "go run ./bundle/internal/schema/*.go ./bundle/schema/jsonschema.json",
+            "go run ./bundle/internal/schema/*.go ./bundle/generated/jsonschema.json",
             "echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes",
             "echo 'go.sum linguist-generated=true' >> ./.gitattributes",
-            "echo 'bundle/schema/jsonschema.json linguist-generated=true' >> ./.gitattributes"
+            "echo 'bundle/generated/jsonschema.json linguist-generated=true' >> ./.gitattributes"
         ]
     }
 }

.codegen/lookup.go.tmpl

@@ -116,10 +116,6 @@ func allResolvers() *resolvers {
 	{{range .Services -}}
 	{{- if in $allowlist .KebabName -}}
 	r.{{.Singular.PascalName}} = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["{{.Singular.PascalName}}"]
-		if ok {
-			return fn(ctx, w, name)
-		}
 		entity, err := w.{{.PascalName}}.GetBy{{range .NamedIdMap.NamePath}}{{.PascalName}}{{end}}(ctx, name)
 		if err != nil {
 			return "", err

.gitattributes

@@ -120,4 +120,4 @@ cmd/workspace/workspace-conf/workspace-conf.go linguist-generated=true
 cmd/workspace/workspace/workspace.go linguist-generated=true
 bundle/internal/tf/schema/\*.go linguist-generated=true
 go.sum linguist-generated=true
-bundle/schema/jsonschema.json linguist-generated=true
+bundle/generated/jsonschema.json linguist-generated=true

bundle/config/mutator/resolve_resource_references_test.go

@@ -2,6 +2,7 @@ package mutator
 
 import (
 	"context"
+	"fmt"
 	"testing"
 
 	"github.com/databricks/cli/bundle"
@@ -43,13 +44,11 @@ func TestResolveClusterReference(t *testing.T) {
 	m := mocks.NewMockWorkspaceClient(t)
 	b.SetWorkpaceClient(m.WorkspaceClient)
 	clusterApi := m.GetMockClustersAPI()
-	clusterApi.EXPECT().ListAll(mock.Anything, compute.ListClustersRequest{
+	clusterApi.EXPECT().GetByClusterName(mock.Anything, clusterRef1).Return(&compute.ClusterDetails{
-		FilterBy: &compute.ListClustersFilterBy{
+		ClusterId: "1234-5678-abcd",
-			ClusterSources: []compute.ClusterSource{compute.ClusterSourceApi, compute.ClusterSourceUi},
+	}, nil)
-		},
+	clusterApi.EXPECT().GetByClusterName(mock.Anything, clusterRef2).Return(&compute.ClusterDetails{
-	}).Return([]compute.ClusterDetails{
+		ClusterId: "9876-5432-xywz",
-		{ClusterId: "1234-5678-abcd", ClusterName: clusterRef1},
-		{ClusterId: "9876-5432-xywz", ClusterName: clusterRef2},
 	}, nil)
 
 	diags := bundle.Apply(context.Background(), b, ResolveResourceReferences())
@@ -79,16 +78,10 @@ func TestResolveNonExistentClusterReference(t *testing.T) {
 	m := mocks.NewMockWorkspaceClient(t)
 	b.SetWorkpaceClient(m.WorkspaceClient)
 	clusterApi := m.GetMockClustersAPI()
-	clusterApi.EXPECT().ListAll(mock.Anything, compute.ListClustersRequest{
+	clusterApi.EXPECT().GetByClusterName(mock.Anything, clusterRef).Return(nil, fmt.Errorf("ClusterDetails named '%s' does not exist", clusterRef))
-		FilterBy: &compute.ListClustersFilterBy{
-			ClusterSources: []compute.ClusterSource{compute.ClusterSourceApi, compute.ClusterSourceUi},
-		},
-	}).Return([]compute.ClusterDetails{
-		{ClusterId: "1234-5678-abcd", ClusterName: "some other cluster"},
-	}, nil)
 
 	diags := bundle.Apply(context.Background(), b, ResolveResourceReferences())
-	require.ErrorContains(t, diags.Error(), "failed to resolve cluster: Random, err: cluster named 'Random' does not exist")
+	require.ErrorContains(t, diags.Error(), "failed to resolve cluster: Random, err: ClusterDetails named 'Random' does not exist")
 }
 
 func TestNoLookupIfVariableIsSet(t *testing.T) {
@@ -165,14 +158,8 @@ func TestResolveVariableReferencesInVariableLookups(t *testing.T) {
 	m := mocks.NewMockWorkspaceClient(t)
 	b.SetWorkpaceClient(m.WorkspaceClient)
 	clusterApi := m.GetMockClustersAPI()
-
+	clusterApi.EXPECT().GetByClusterName(mock.Anything, "cluster-bar-dev").Return(&compute.ClusterDetails{
-	clusterApi.EXPECT().ListAll(mock.Anything, compute.ListClustersRequest{
+		ClusterId: "1234-5678-abcd",
-		FilterBy: &compute.ListClustersFilterBy{
-			ClusterSources: []compute.ClusterSource{compute.ClusterSourceApi, compute.ClusterSourceUi},
-		},
-	}).Return([]compute.ClusterDetails{
-		{ClusterId: "1234-5678-abcd", ClusterName: "cluster-bar-dev"},
-		{ClusterId: "9876-5432-xywz", ClusterName: "some other cluster"},
 	}, nil)
 
 	diags := bundle.Apply(context.Background(), b, bundle.Seq(ResolveVariableReferencesInLookup(), ResolveResourceReferences()))

bundle/config/variable/lookup.go

@@ -220,10 +220,6 @@ type resolvers struct {
 func allResolvers() *resolvers {
 	r := &resolvers{}
 	r.Alert = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["Alert"]
-		if ok {
-			return fn(ctx, w, name)
-		}
 		entity, err := w.Alerts.GetByDisplayName(ctx, name)
 		if err != nil {
 			return "", err
@@ -232,10 +228,6 @@ func allResolvers() *resolvers {
 		return fmt.Sprint(entity.Id), nil
 	}
 	r.ClusterPolicy = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["ClusterPolicy"]
-		if ok {
-			return fn(ctx, w, name)
-		}
 		entity, err := w.ClusterPolicies.GetByName(ctx, name)
 		if err != nil {
 			return "", err
@@ -244,10 +236,6 @@ func allResolvers() *resolvers {
 		return fmt.Sprint(entity.PolicyId), nil
 	}
 	r.Cluster = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["Cluster"]
-		if ok {
-			return fn(ctx, w, name)
-		}
 		entity, err := w.Clusters.GetByClusterName(ctx, name)
 		if err != nil {
 			return "", err
@@ -256,10 +244,6 @@ func allResolvers() *resolvers {
 		return fmt.Sprint(entity.ClusterId), nil
 	}
 	r.Dashboard = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["Dashboard"]
-		if ok {
-			return fn(ctx, w, name)
-		}
 		entity, err := w.Dashboards.GetByName(ctx, name)
 		if err != nil {
 			return "", err
@@ -268,10 +252,6 @@ func allResolvers() *resolvers {
 		return fmt.Sprint(entity.Id), nil
 	}
 	r.InstancePool = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["InstancePool"]
-		if ok {
-			return fn(ctx, w, name)
-		}
 		entity, err := w.InstancePools.GetByInstancePoolName(ctx, name)
 		if err != nil {
 			return "", err
@@ -280,10 +260,6 @@ func allResolvers() *resolvers {
 		return fmt.Sprint(entity.InstancePoolId), nil
 	}
 	r.Job = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["Job"]
-		if ok {
-			return fn(ctx, w, name)
-		}
 		entity, err := w.Jobs.GetBySettingsName(ctx, name)
 		if err != nil {
 			return "", err
@@ -292,10 +268,6 @@ func allResolvers() *resolvers {
 		return fmt.Sprint(entity.JobId), nil
 	}
 	r.Metastore = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["Metastore"]
-		if ok {
-			return fn(ctx, w, name)
-		}
 		entity, err := w.Metastores.GetByName(ctx, name)
 		if err != nil {
 			return "", err
@@ -304,10 +276,6 @@ func allResolvers() *resolvers {
 		return fmt.Sprint(entity.MetastoreId), nil
 	}
 	r.Pipeline = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["Pipeline"]
-		if ok {
-			return fn(ctx, w, name)
-		}
 		entity, err := w.Pipelines.GetByName(ctx, name)
 		if err != nil {
 			return "", err
@@ -316,10 +284,6 @@ func allResolvers() *resolvers {
 		return fmt.Sprint(entity.PipelineId), nil
 	}
 	r.Query = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["Query"]
-		if ok {
-			return fn(ctx, w, name)
-		}
 		entity, err := w.Queries.GetByDisplayName(ctx, name)
 		if err != nil {
 			return "", err
@@ -328,10 +292,6 @@ func allResolvers() *resolvers {
 		return fmt.Sprint(entity.Id), nil
 	}
 	r.ServicePrincipal = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["ServicePrincipal"]
-		if ok {
-			return fn(ctx, w, name)
-		}
 		entity, err := w.ServicePrincipals.GetByDisplayName(ctx, name)
 		if err != nil {
 			return "", err
@@ -340,10 +300,6 @@ func allResolvers() *resolvers {
 		return fmt.Sprint(entity.ApplicationId), nil
 	}
 	r.Warehouse = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-		fn, ok := lookupOverrides["Warehouse"]
-		if ok {
-			return fn(ctx, w, name)
-		}
 		entity, err := w.Warehouses.GetByName(ctx, name)
 		if err != nil {
 			return "", err

bundle/config/variable/lookup_overrides.go

@@ -1,41 +0,0 @@
-package variable
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/databricks/databricks-sdk-go"
-	"github.com/databricks/databricks-sdk-go/service/compute"
-)
-
-var lookupOverrides = map[string]resolverFunc{
-	"Cluster": resolveCluster,
-}
-
-// We added a custom resolver for the cluster to add filtering for the cluster source when we list all clusters.
-// Without the filtering listing could take a very long time (5-10 mins) which leads to lookup timeouts.
-func resolveCluster(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
-	result, err := w.Clusters.ListAll(ctx, compute.ListClustersRequest{
-		FilterBy: &compute.ListClustersFilterBy{
-			ClusterSources: []compute.ClusterSource{compute.ClusterSourceApi, compute.ClusterSourceUi},
-		},
-	})
-
-	if err != nil {
-		return "", err
-	}
-
-	tmp := map[string][]compute.ClusterDetails{}
-	for _, v := range result {
-		key := v.ClusterName
-		tmp[key] = append(tmp[key], v)
-	}
-	alternatives, ok := tmp[name]
-	if !ok || len(alternatives) == 0 {
-		return "", fmt.Errorf("cluster named '%s' does not exist", name)
-	}
-	if len(alternatives) > 1 {
-		return "", fmt.Errorf("there are %d instances of clusters named '%s'", len(alternatives), name)
-	}
-	return alternatives[0].ClusterId, nil
-}

bundle/libraries/expand_glob_references.go

@@ -39,7 +39,7 @@ func getLibDetails(v dyn.Value) (string, string, bool) {
 }
 
 func findMatches(b *bundle.Bundle, path string) ([]string, error) {
-	matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, path))
+	matches, err := filepath.Glob(filepath.Join(b.RootPath, path))
 	if err != nil {
 		return nil, err
 	}
@@ -52,10 +52,10 @@ func findMatches(b *bundle.Bundle, path string) ([]string, error) {
 		}
 	}
 
-	// We make the matched path relative to the sync root path before storing it
+	// We make the matched path relative to the root path before storing it
 	// to allow upload mutator to distinguish between local and remote paths
 	for i, match := range matches {
-		matches[i], err = filepath.Rel(b.SyncRootPath, match)
+		matches[i], err = filepath.Rel(b.RootPath, match)
 		if err != nil {
 			return nil, err
 		}
@@ -211,8 +211,8 @@ func (e *expand) Name() string {
 
 // ExpandGlobReferences expands any glob references in the libraries or environments section
 // to corresponding local paths.
-// We only expand local paths (i.e. paths that are relative to the sync root path).
+// We only expand local paths (i.e. paths that are relative to the root path).
-// After expanding we make the paths relative to the sync root path to allow upload mutator later in the chain to
+// After expanding we make the paths relative to the root path to allow upload mutator later in the chain to
 // distinguish between local and remote paths.
 func ExpandGlobReferences() bundle.Mutator {
 	return &expand{}

bundle/libraries/expand_glob_references_test.go

@@ -23,7 +23,7 @@ func TestGlobReferencesExpandedForTaskLibraries(t *testing.T) {
 	testutil.Touch(t, dir, "jar", "my2.jar")
 
 	b := &bundle.Bundle{
-		SyncRootPath: dir,
+		RootPath: dir,
 		Config: config.Root{
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
@@ -104,7 +104,7 @@ func TestGlobReferencesExpandedForForeachTaskLibraries(t *testing.T) {
 	testutil.Touch(t, dir, "jar", "my2.jar")
 
 	b := &bundle.Bundle{
-		SyncRootPath: dir,
+		RootPath: dir,
 		Config: config.Root{
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
@@ -189,7 +189,7 @@ func TestGlobReferencesExpandedForEnvironmentsDeps(t *testing.T) {
 	testutil.Touch(t, dir, "jar", "my2.jar")
 
 	b := &bundle.Bundle{
-		SyncRootPath: dir,
+		RootPath: dir,
 		Config: config.Root{
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{

bundle/libraries/match_test.go

@@ -18,7 +18,7 @@ func TestValidateEnvironments(t *testing.T) {
 	testutil.Touch(t, tmpDir, "wheel.whl")
 
 	b := &bundle.Bundle{
-		SyncRootPath: tmpDir,
+		RootPath: tmpDir,
 		Config: config.Root{
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
@@ -50,7 +50,7 @@ func TestValidateEnvironmentsNoFile(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	b := &bundle.Bundle{
-		SyncRootPath: tmpDir,
+		RootPath: tmpDir,
 		Config: config.Root{
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
@@ -84,7 +84,7 @@ func TestValidateTaskLibraries(t *testing.T) {
 	testutil.Touch(t, tmpDir, "wheel.whl")
 
 	b := &bundle.Bundle{
-		SyncRootPath: tmpDir,
+		RootPath: tmpDir,
 		Config: config.Root{
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
@@ -117,7 +117,7 @@ func TestValidateTaskLibrariesNoFile(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	b := &bundle.Bundle{
-		SyncRootPath: tmpDir,
+		RootPath: tmpDir,
 		Config: config.Root{
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{

bundle/libraries/upload.go

@@ -74,7 +74,7 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]configLocation, error
 					return v, nil
 				}
 
-				source = filepath.Join(b.SyncRootPath, source)
+				source = filepath.Join(b.RootPath, source)
 				libs[source] = append(libs[source], configLocation{
 					configPath: p,
 					location:   v.Location(),

bundle/libraries/upload_test.go

@@ -24,7 +24,7 @@ func TestArtifactUploadForWorkspace(t *testing.T) {
 	whlLocalPath := filepath.Join(whlFolder, "source.whl")
 
 	b := &bundle.Bundle{
-		SyncRootPath: tmpDir,
+		RootPath: tmpDir,
 		Config: config.Root{
 			Workspace: config.Workspace{
 				ArtifactPath: "/foo/bar/artifacts",
@@ -112,7 +112,7 @@ func TestArtifactUploadForVolumes(t *testing.T) {
 	whlLocalPath := filepath.Join(whlFolder, "source.whl")
 
 	b := &bundle.Bundle{
-		SyncRootPath: tmpDir,
+		RootPath: tmpDir,
 		Config: config.Root{
 			Workspace: config.Workspace{
 				ArtifactPath: "/Volumes/foo/bar/artifacts",
@@ -200,7 +200,7 @@ func TestArtifactUploadWithNoLibraryReference(t *testing.T) {
 	whlLocalPath := filepath.Join(whlFolder, "source.whl")
 
 	b := &bundle.Bundle{
-		SyncRootPath: tmpDir,
+		RootPath: tmpDir,
 		Config: config.Root{
 			Workspace: config.Workspace{
 				ArtifactPath: "/Workspace/foo/bar/artifacts",
@@ -240,7 +240,7 @@ func TestUploadMultipleLibraries(t *testing.T) {
 	testutil.Touch(t, whlFolder, "source4.whl")
 
 	b := &bundle.Bundle{
-		SyncRootPath: tmpDir,
+		RootPath: tmpDir,
 		Config: config.Root{
 			Workspace: config.Workspace{
 				ArtifactPath: "/foo/bar/artifacts",

bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml

@@ -1,9 +1,6 @@
 bundle:
   name: python-wheel-local
 
-workspace:
-  artifact_path: /foo/bar
-
 resources:
   jobs:
     test_job:

bundle/tests/python_wheel_test.go

@@ -15,10 +15,11 @@ import (
 )
 
 func TestPythonWheelBuild(t *testing.T) {
-	b := loadTarget(t, "./python_wheel/python_wheel", "default")
-
 	ctx := context.Background()
-	diags := bundle.Apply(ctx, b, phases.Build())
+	b, err := bundle.Load(ctx, "./python_wheel/python_wheel")
+	require.NoError(t, err)
+
+	diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build()))
 	require.NoError(t, diags.Error())
 
 	matches, err := filepath.Glob("./python_wheel/python_wheel/my_test_code/dist/my_test_code-*.whl")
@@ -31,10 +32,11 @@ func TestPythonWheelBuild(t *testing.T) {
 }
 
 func TestPythonWheelBuildAutoDetect(t *testing.T) {
-	b := loadTarget(t, "./python_wheel/python_wheel_no_artifact", "default")
-
 	ctx := context.Background()
-	diags := bundle.Apply(ctx, b, phases.Build())
+	b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_artifact")
+	require.NoError(t, err)
+
+	diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build()))
 	require.NoError(t, diags.Error())
 
 	matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact/dist/my_test_code-*.whl")
@@ -47,10 +49,11 @@ func TestPythonWheelBuildAutoDetect(t *testing.T) {
 }
 
 func TestPythonWheelBuildAutoDetectWithNotebookTask(t *testing.T) {
-	b := loadTarget(t, "./python_wheel/python_wheel_no_artifact_notebook", "default")
-
 	ctx := context.Background()
-	diags := bundle.Apply(ctx, b, phases.Build())
+	b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_artifact_notebook")
+	require.NoError(t, err)
+
+	diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build()))
 	require.NoError(t, diags.Error())
 
 	matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact_notebook/dist/my_test_code-*.whl")
@@ -63,10 +66,11 @@ func TestPythonWheelBuildAutoDetectWithNotebookTask(t *testing.T) {
 }
 
 func TestPythonWheelWithDBFSLib(t *testing.T) {
-	b := loadTarget(t, "./python_wheel/python_wheel_dbfs_lib", "default")
-
 	ctx := context.Background()
-	diags := bundle.Apply(ctx, b, phases.Build())
+	b, err := bundle.Load(ctx, "./python_wheel/python_wheel_dbfs_lib")
+	require.NoError(t, err)
+
+	diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build()))
 	require.NoError(t, diags.Error())
 
 	match := libraries.ExpandGlobReferences()
@@ -75,11 +79,11 @@ func TestPythonWheelWithDBFSLib(t *testing.T) {
 }
 
 func TestPythonWheelBuildNoBuildJustUpload(t *testing.T) {
-	b := loadTarget(t, "./python_wheel/python_wheel_no_artifact_no_setup", "default")
-
 	ctx := context.Background()
-	diags := bundle.Apply(ctx, b, phases.Build())
+	b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_artifact_no_setup")
-	require.NoError(t, diags.Error())
+	require.NoError(t, err)
+
+	b.Config.Workspace.ArtifactPath = "/foo/bar"
 
 	mockFiler := mockfiler.NewMockFiler(t)
 	mockFiler.EXPECT().Write(
@@ -90,20 +94,20 @@ func TestPythonWheelBuildNoBuildJustUpload(t *testing.T) {
 		filer.CreateParentDirectories,
 	).Return(nil)
 
-	diags = bundle.Apply(ctx, b, bundle.Seq(
+	u := libraries.UploadWithClient(mockFiler)
-		libraries.ExpandGlobReferences(),
+	diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build(), libraries.ExpandGlobReferences(), u))
-		libraries.UploadWithClient(mockFiler),
-	))
 	require.NoError(t, diags.Error())
 	require.Empty(t, diags)
+
 	require.Equal(t, "/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[0].Libraries[0].Whl)
 }
 
 func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) {
-	b := loadTarget(t, "./python_wheel/environment_key", "default")
-
 	ctx := context.Background()
-	diags := bundle.Apply(ctx, b, phases.Build())
+	b, err := bundle.Load(ctx, "./python_wheel/environment_key")
+	require.NoError(t, err)
+
+	diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build()))
 	require.NoError(t, diags.Error())
 
 	matches, err := filepath.Glob("./python_wheel/environment_key/my_test_code/dist/my_test_code-*.whl")
@@ -116,10 +120,11 @@ func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) {
 }
 
 func TestPythonWheelBuildMultiple(t *testing.T) {
-	b := loadTarget(t, "./python_wheel/python_wheel_multiple", "default")
-
 	ctx := context.Background()
-	diags := bundle.Apply(ctx, b, phases.Build())
+	b, err := bundle.Load(ctx, "./python_wheel/python_wheel_multiple")
+	require.NoError(t, err)
+
+	diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build()))
 	require.NoError(t, diags.Error())
 
 	matches, err := filepath.Glob("./python_wheel/python_wheel_multiple/my_test_code/dist/my_test_code*.whl")
@@ -132,10 +137,11 @@ func TestPythonWheelBuildMultiple(t *testing.T) {
 }
 
 func TestPythonWheelNoBuild(t *testing.T) {
-	b := loadTarget(t, "./python_wheel/python_wheel_no_build", "default")
-
 	ctx := context.Background()
-	diags := bundle.Apply(ctx, b, phases.Build())
+	b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_build")
+	require.NoError(t, err)
+
+	diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build()))
 	require.NoError(t, diags.Error())
 
 	match := libraries.ExpandGlobReferences()

bundle/tests/variables_test.go

@@ -124,13 +124,8 @@ func TestVariablesWithTargetLookupOverrides(t *testing.T) {
 	}, nil)
 
 	clustersApi := mockWorkspaceClient.GetMockClustersAPI()
-	clustersApi.EXPECT().ListAll(mock.Anything, compute.ListClustersRequest{
+	clustersApi.EXPECT().GetByClusterName(mock.Anything, "some-test-cluster").Return(&compute.ClusterDetails{
-		FilterBy: &compute.ListClustersFilterBy{
+		ClusterId: "4321",
-			ClusterSources: []compute.ClusterSource{compute.ClusterSourceApi, compute.ClusterSourceUi},
-		},
-	}).Return([]compute.ClusterDetails{
-		{ClusterId: "4321", ClusterName: "some-test-cluster"},
-		{ClusterId: "9876", ClusterName: "some-other-cluster"},
 	}, nil)
 
 	clusterPoliciesApi := mockWorkspaceClient.GetMockClusterPoliciesAPI()

cmd/bundle/schema.go

@@ -1,6 +1,8 @@
 package bundle
 
 import (
+	_ "embed"
+
 	"github.com/databricks/cli/bundle/schema"
 	"github.com/databricks/cli/cmd/root"
 	"github.com/spf13/cobra"

libs/jsonschema/from_type.go

@@ -203,10 +203,6 @@ func (c *constructor) walk(typ reflect.Type) (string, error) {
 // This function returns all member fields of the provided type.
 // If the type has embedded (aka anonymous) fields, this function traverses
 // those in a breadth first manner
-//
-// BFS is important because we want the a field defined at a higher level embedded
-// struct to be given preference over a field with the same name defined at a lower
-// level embedded struct. For example see: TestHigherLevelEmbeddedFieldIsInSchema
 func getStructFields(typ reflect.Type) []reflect.StructField {
 	fields := []reflect.StructField{}
 	bfsQueue := list.New()
@@ -276,13 +272,6 @@ func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) {
 			continue
 		}
 
-		// Skip property if it is already present in the schema.
-		// This can happen if the same field is defined multiple times across
-		// a tree of embedded structs. For example see: TestHigherLevelEmbeddedFieldIsInSchema
-		if _, ok := res.Properties[fieldName]; ok {
-			continue
-		}
-
 		// "omitempty" tags in the Go SDK structs represent fields that not are
 		// required to be present in the API payload. Thus its absence in the
 		// tags list indicates that the field is required.

libs/jsonschema/from_type_test.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/databricks/cli/libs/jsonschema/test_types"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func TestFromTypeBasic(t *testing.T) {
@@ -167,46 +166,6 @@ func TestGetStructFields(t *testing.T) {
 	assert.Equal(t, "InnerField", fields[3].Name)
 }
 
-func TestHigherLevelEmbeddedFieldIsInSchema(t *testing.T) {
-	type Inner struct {
-		Override string `json:"override,omitempty"`
-	}
-
-	type EmbeddedOne struct {
-		Inner
-	}
-
-	type EmbeddedTwo struct {
-		Override int `json:"override,omitempty"`
-	}
-
-	type Outer struct {
-		EmbeddedOne
-		EmbeddedTwo
-	}
-
-	intRef := "#/$defs/int"
-	expected := Schema{
-		Type: "object",
-		Definitions: map[string]any{
-			"int": Schema{
-				Type: "integer",
-			},
-		},
-		Properties: map[string]*Schema{
-			"override": {
-				Reference: &intRef,
-			},
-		},
-		AdditionalProperties: false,
-		Required:             []string{},
-	}
-
-	s, err := FromType(reflect.TypeOf(Outer{}), nil)
-	require.NoError(t, err)
-	assert.Equal(t, expected, s)
-}
-
 func TestFromTypeNested(t *testing.T) {
 	type Inner struct {
 		S string `json:"s"`