From 02e83877f4284589942bb439e849e01fc81a4314 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 6 Sep 2024 13:34:57 +0200 Subject: [PATCH 1/2] Added listing cluster filtering for cluster lookups (#1754) ## Changes We added a custom resolver for the cluster to add filtering for the cluster source when we list all clusters. Without the filtering listing could take a very long time (5-10 mins) which leads to lookup timeouts. ## Tests Existing unit tests passing --- .codegen/lookup.go.tmpl | 4 ++ .../resolve_resource_references_test.go | 33 +++++++++----- bundle/config/variable/lookup.go | 44 +++++++++++++++++++ bundle/config/variable/lookup_overrides.go | 41 +++++++++++++++++ bundle/tests/variables_test.go | 9 +++- 5 files changed, 119 insertions(+), 12 deletions(-) create mode 100644 bundle/config/variable/lookup_overrides.go diff --git a/.codegen/lookup.go.tmpl b/.codegen/lookup.go.tmpl index 431709f90..124b629d0 100644 --- a/.codegen/lookup.go.tmpl +++ b/.codegen/lookup.go.tmpl @@ -116,6 +116,10 @@ func allResolvers() *resolvers { {{range .Services -}} {{- if in $allowlist .KebabName -}} r.{{.Singular.PascalName}} = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["{{.Singular.PascalName}}"] + if ok { + return fn(ctx, w, name) + } entity, err := w.{{.PascalName}}.GetBy{{range .NamedIdMap.NamePath}}{{.PascalName}}{{end}}(ctx, name) if err != nil { return "", err diff --git a/bundle/config/mutator/resolve_resource_references_test.go b/bundle/config/mutator/resolve_resource_references_test.go index 86a03b23e..ee2f0e2ea 100644 --- a/bundle/config/mutator/resolve_resource_references_test.go +++ b/bundle/config/mutator/resolve_resource_references_test.go @@ -2,7 +2,6 @@ package mutator import ( "context" - "fmt" "testing" "github.com/databricks/cli/bundle" @@ -44,11 +43,13 @@ func TestResolveClusterReference(t *testing.T) { m := mocks.NewMockWorkspaceClient(t) b.SetWorkpaceClient(m.WorkspaceClient) clusterApi := m.GetMockClustersAPI() - clusterApi.EXPECT().GetByClusterName(mock.Anything, clusterRef1).Return(&compute.ClusterDetails{ - ClusterId: "1234-5678-abcd", - }, nil) - clusterApi.EXPECT().GetByClusterName(mock.Anything, clusterRef2).Return(&compute.ClusterDetails{ - ClusterId: "9876-5432-xywz", + clusterApi.EXPECT().ListAll(mock.Anything, compute.ListClustersRequest{ + FilterBy: &compute.ListClustersFilterBy{ + ClusterSources: []compute.ClusterSource{compute.ClusterSourceApi, compute.ClusterSourceUi}, + }, + }).Return([]compute.ClusterDetails{ + {ClusterId: "1234-5678-abcd", ClusterName: clusterRef1}, + {ClusterId: "9876-5432-xywz", ClusterName: clusterRef2}, }, nil) diags := bundle.Apply(context.Background(), b, ResolveResourceReferences()) @@ -78,10 +79,16 @@ func TestResolveNonExistentClusterReference(t *testing.T) { m := mocks.NewMockWorkspaceClient(t) b.SetWorkpaceClient(m.WorkspaceClient) clusterApi := m.GetMockClustersAPI() - clusterApi.EXPECT().GetByClusterName(mock.Anything, clusterRef).Return(nil, fmt.Errorf("ClusterDetails named '%s' does not exist", clusterRef)) + clusterApi.EXPECT().ListAll(mock.Anything, compute.ListClustersRequest{ + FilterBy: &compute.ListClustersFilterBy{ + ClusterSources: []compute.ClusterSource{compute.ClusterSourceApi, compute.ClusterSourceUi}, + }, + }).Return([]compute.ClusterDetails{ + {ClusterId: "1234-5678-abcd", ClusterName: "some other cluster"}, + }, nil) diags := bundle.Apply(context.Background(), b, ResolveResourceReferences()) - require.ErrorContains(t, diags.Error(), "failed to resolve cluster: Random, err: ClusterDetails named 'Random' does not exist") + require.ErrorContains(t, diags.Error(), "failed to resolve cluster: Random, err: cluster named 'Random' does not exist") } func TestNoLookupIfVariableIsSet(t *testing.T) { @@ -158,8 +165,14 @@ func TestResolveVariableReferencesInVariableLookups(t *testing.T) { m := mocks.NewMockWorkspaceClient(t) b.SetWorkpaceClient(m.WorkspaceClient) clusterApi := m.GetMockClustersAPI() - clusterApi.EXPECT().GetByClusterName(mock.Anything, "cluster-bar-dev").Return(&compute.ClusterDetails{ - ClusterId: "1234-5678-abcd", + + clusterApi.EXPECT().ListAll(mock.Anything, compute.ListClustersRequest{ + FilterBy: &compute.ListClustersFilterBy{ + ClusterSources: []compute.ClusterSource{compute.ClusterSourceApi, compute.ClusterSourceUi}, + }, + }).Return([]compute.ClusterDetails{ + {ClusterId: "1234-5678-abcd", ClusterName: "cluster-bar-dev"}, + {ClusterId: "9876-5432-xywz", ClusterName: "some other cluster"}, }, nil) diags := bundle.Apply(context.Background(), b, bundle.Seq(ResolveVariableReferencesInLookup(), ResolveResourceReferences())) diff --git a/bundle/config/variable/lookup.go b/bundle/config/variable/lookup.go index 9c85e2a71..e40b0ef7a 100755 --- a/bundle/config/variable/lookup.go +++ b/bundle/config/variable/lookup.go @@ -220,6 +220,10 @@ type resolvers struct { func allResolvers() *resolvers { r := &resolvers{} r.Alert = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["Alert"] + if ok { + return fn(ctx, w, name) + } entity, err := w.Alerts.GetByDisplayName(ctx, name) if err != nil { return "", err @@ -228,6 +232,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.Id), nil } r.ClusterPolicy = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["ClusterPolicy"] + if ok { + return fn(ctx, w, name) + } entity, err := w.ClusterPolicies.GetByName(ctx, name) if err != nil { return "", err @@ -236,6 +244,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.PolicyId), nil } r.Cluster = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["Cluster"] + if ok { + return fn(ctx, w, name) + } entity, err := w.Clusters.GetByClusterName(ctx, name) if err != nil { return "", err @@ -244,6 +256,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.ClusterId), nil } r.Dashboard = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["Dashboard"] + if ok { + return fn(ctx, w, name) + } entity, err := w.Dashboards.GetByName(ctx, name) if err != nil { return "", err @@ -252,6 +268,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.Id), nil } r.InstancePool = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["InstancePool"] + if ok { + return fn(ctx, w, name) + } entity, err := w.InstancePools.GetByInstancePoolName(ctx, name) if err != nil { return "", err @@ -260,6 +280,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.InstancePoolId), nil } r.Job = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["Job"] + if ok { + return fn(ctx, w, name) + } entity, err := w.Jobs.GetBySettingsName(ctx, name) if err != nil { return "", err @@ -268,6 +292,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.JobId), nil } r.Metastore = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["Metastore"] + if ok { + return fn(ctx, w, name) + } entity, err := w.Metastores.GetByName(ctx, name) if err != nil { return "", err @@ -276,6 +304,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.MetastoreId), nil } r.Pipeline = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["Pipeline"] + if ok { + return fn(ctx, w, name) + } entity, err := w.Pipelines.GetByName(ctx, name) if err != nil { return "", err @@ -284,6 +316,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.PipelineId), nil } r.Query = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["Query"] + if ok { + return fn(ctx, w, name) + } entity, err := w.Queries.GetByDisplayName(ctx, name) if err != nil { return "", err @@ -292,6 +328,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.Id), nil } r.ServicePrincipal = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["ServicePrincipal"] + if ok { + return fn(ctx, w, name) + } entity, err := w.ServicePrincipals.GetByDisplayName(ctx, name) if err != nil { return "", err @@ -300,6 +340,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.ApplicationId), nil } r.Warehouse = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["Warehouse"] + if ok { + return fn(ctx, w, name) + } entity, err := w.Warehouses.GetByName(ctx, name) if err != nil { return "", err diff --git a/bundle/config/variable/lookup_overrides.go b/bundle/config/variable/lookup_overrides.go new file mode 100644 index 000000000..1be373dc6 --- /dev/null +++ b/bundle/config/variable/lookup_overrides.go @@ -0,0 +1,41 @@ +package variable + +import ( + "context" + "fmt" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +var lookupOverrides = map[string]resolverFunc{ + "Cluster": resolveCluster, +} + +// We added a custom resolver for the cluster to add filtering for the cluster source when we list all clusters. +// Without the filtering listing could take a very long time (5-10 mins) which leads to lookup timeouts. +func resolveCluster(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + result, err := w.Clusters.ListAll(ctx, compute.ListClustersRequest{ + FilterBy: &compute.ListClustersFilterBy{ + ClusterSources: []compute.ClusterSource{compute.ClusterSourceApi, compute.ClusterSourceUi}, + }, + }) + + if err != nil { + return "", err + } + + tmp := map[string][]compute.ClusterDetails{} + for _, v := range result { + key := v.ClusterName + tmp[key] = append(tmp[key], v) + } + alternatives, ok := tmp[name] + if !ok || len(alternatives) == 0 { + return "", fmt.Errorf("cluster named '%s' does not exist", name) + } + if len(alternatives) > 1 { + return "", fmt.Errorf("there are %d instances of clusters named '%s'", len(alternatives), name) + } + return alternatives[0].ClusterId, nil +} diff --git a/bundle/tests/variables_test.go b/bundle/tests/variables_test.go index 51a23e5d5..9451c5a04 100644 --- a/bundle/tests/variables_test.go +++ b/bundle/tests/variables_test.go @@ -124,8 +124,13 @@ func TestVariablesWithTargetLookupOverrides(t *testing.T) { }, nil) clustersApi := mockWorkspaceClient.GetMockClustersAPI() - clustersApi.EXPECT().GetByClusterName(mock.Anything, "some-test-cluster").Return(&compute.ClusterDetails{ - ClusterId: "4321", + clustersApi.EXPECT().ListAll(mock.Anything, compute.ListClustersRequest{ + FilterBy: &compute.ListClustersFilterBy{ + ClusterSources: []compute.ClusterSource{compute.ClusterSourceApi, compute.ClusterSourceUi}, + }, + }).Return([]compute.ClusterDetails{ + {ClusterId: "4321", ClusterName: "some-test-cluster"}, + {ClusterId: "9876", ClusterName: "some-other-cluster"}, }, nil) clusterPoliciesApi := mockWorkspaceClient.GetMockClusterPoliciesAPI() From b451905b6ea3091fac34775e3a14a5af9f649d53 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 9 Sep 2024 11:56:16 +0200 Subject: [PATCH 2/2] Expand library globs relative to the sync root (#1756) ## Changes Library glob expansion happens during deployment. Before that, all entries that refer to local paths in resource definitions are made relative to the _sync root_. Before #1694, they were made relative to the _bundle root_. This PR didn't update the library glob expansion code to use the sync root path. If you were using the sync paths setting with library globs, the CLI would fail to expand the globs because the code was using the wrong path to anchor those globs. This change fixes the issue. ## Tests Manually confirmed that this fixes the issue reported in #1755. --- bundle/libraries/expand_glob_references.go | 10 +-- .../libraries/expand_glob_references_test.go | 6 +- bundle/libraries/match_test.go | 8 +-- bundle/libraries/upload.go | 2 +- bundle/libraries/upload_test.go | 8 +-- .../bundle.yml | 3 + bundle/tests/python_wheel_test.go | 64 +++++++++---------- 7 files changed, 49 insertions(+), 52 deletions(-) diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index 9322a06b8..c71615e0e 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -39,7 +39,7 @@ func getLibDetails(v dyn.Value) (string, string, bool) { } func findMatches(b *bundle.Bundle, path string) ([]string, error) { - matches, err := filepath.Glob(filepath.Join(b.RootPath, path)) + matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, path)) if err != nil { return nil, err } @@ -52,10 +52,10 @@ func findMatches(b *bundle.Bundle, path string) ([]string, error) { } } - // We make the matched path relative to the root path before storing it + // We make the matched path relative to the sync root path before storing it // to allow upload mutator to distinguish between local and remote paths for i, match := range matches { - matches[i], err = filepath.Rel(b.RootPath, match) + matches[i], err = filepath.Rel(b.SyncRootPath, match) if err != nil { return nil, err } @@ -211,8 +211,8 @@ func (e *expand) Name() string { // ExpandGlobReferences expands any glob references in the libraries or environments section // to corresponding local paths. -// We only expand local paths (i.e. paths that are relative to the root path). -// After expanding we make the paths relative to the root path to allow upload mutator later in the chain to +// We only expand local paths (i.e. paths that are relative to the sync root path). +// After expanding we make the paths relative to the sync root path to allow upload mutator later in the chain to // distinguish between local and remote paths. func ExpandGlobReferences() bundle.Mutator { return &expand{} diff --git a/bundle/libraries/expand_glob_references_test.go b/bundle/libraries/expand_glob_references_test.go index 34855b539..e7f2e1693 100644 --- a/bundle/libraries/expand_glob_references_test.go +++ b/bundle/libraries/expand_glob_references_test.go @@ -23,7 +23,7 @@ func TestGlobReferencesExpandedForTaskLibraries(t *testing.T) { testutil.Touch(t, dir, "jar", "my2.jar") b := &bundle.Bundle{ - RootPath: dir, + SyncRootPath: dir, Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -104,7 +104,7 @@ func TestGlobReferencesExpandedForForeachTaskLibraries(t *testing.T) { testutil.Touch(t, dir, "jar", "my2.jar") b := &bundle.Bundle{ - RootPath: dir, + SyncRootPath: dir, Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -189,7 +189,7 @@ func TestGlobReferencesExpandedForEnvironmentsDeps(t *testing.T) { testutil.Touch(t, dir, "jar", "my2.jar") b := &bundle.Bundle{ - RootPath: dir, + SyncRootPath: dir, Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ diff --git a/bundle/libraries/match_test.go b/bundle/libraries/match_test.go index e60504c84..78765cbd4 100644 --- a/bundle/libraries/match_test.go +++ b/bundle/libraries/match_test.go @@ -18,7 +18,7 @@ func TestValidateEnvironments(t *testing.T) { testutil.Touch(t, tmpDir, "wheel.whl") b := &bundle.Bundle{ - RootPath: tmpDir, + SyncRootPath: tmpDir, Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -50,7 +50,7 @@ func TestValidateEnvironmentsNoFile(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ - RootPath: tmpDir, + SyncRootPath: tmpDir, Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -84,7 +84,7 @@ func TestValidateTaskLibraries(t *testing.T) { testutil.Touch(t, tmpDir, "wheel.whl") b := &bundle.Bundle{ - RootPath: tmpDir, + SyncRootPath: tmpDir, Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -117,7 +117,7 @@ func TestValidateTaskLibrariesNoFile(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ - RootPath: tmpDir, + SyncRootPath: tmpDir, Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ diff --git a/bundle/libraries/upload.go b/bundle/libraries/upload.go index 224e7ab2d..90a1a21fc 100644 --- a/bundle/libraries/upload.go +++ b/bundle/libraries/upload.go @@ -74,7 +74,7 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]configLocation, error return v, nil } - source = filepath.Join(b.RootPath, source) + source = filepath.Join(b.SyncRootPath, source) libs[source] = append(libs[source], configLocation{ configPath: p, location: v.Location(), diff --git a/bundle/libraries/upload_test.go b/bundle/libraries/upload_test.go index 82fe6e7c7..44b194c56 100644 --- a/bundle/libraries/upload_test.go +++ b/bundle/libraries/upload_test.go @@ -24,7 +24,7 @@ func TestArtifactUploadForWorkspace(t *testing.T) { whlLocalPath := filepath.Join(whlFolder, "source.whl") b := &bundle.Bundle{ - RootPath: tmpDir, + SyncRootPath: tmpDir, Config: config.Root{ Workspace: config.Workspace{ ArtifactPath: "/foo/bar/artifacts", @@ -112,7 +112,7 @@ func TestArtifactUploadForVolumes(t *testing.T) { whlLocalPath := filepath.Join(whlFolder, "source.whl") b := &bundle.Bundle{ - RootPath: tmpDir, + SyncRootPath: tmpDir, Config: config.Root{ Workspace: config.Workspace{ ArtifactPath: "/Volumes/foo/bar/artifacts", @@ -200,7 +200,7 @@ func TestArtifactUploadWithNoLibraryReference(t *testing.T) { whlLocalPath := filepath.Join(whlFolder, "source.whl") b := &bundle.Bundle{ - RootPath: tmpDir, + SyncRootPath: tmpDir, Config: config.Root{ Workspace: config.Workspace{ ArtifactPath: "/Workspace/foo/bar/artifacts", @@ -240,7 +240,7 @@ func TestUploadMultipleLibraries(t *testing.T) { testutil.Touch(t, whlFolder, "source4.whl") b := &bundle.Bundle{ - RootPath: tmpDir, + SyncRootPath: tmpDir, Config: config.Root{ Workspace: config.Workspace{ ArtifactPath: "/foo/bar/artifacts", diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml b/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml index 492861969..d03084303 100644 --- a/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml @@ -1,6 +1,9 @@ bundle: name: python-wheel-local +workspace: + artifact_path: /foo/bar + resources: jobs: test_job: diff --git a/bundle/tests/python_wheel_test.go b/bundle/tests/python_wheel_test.go index c4d85703c..c982c09d6 100644 --- a/bundle/tests/python_wheel_test.go +++ b/bundle/tests/python_wheel_test.go @@ -15,11 +15,10 @@ import ( ) func TestPythonWheelBuild(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./python_wheel/python_wheel") - require.NoError(t, err) + b := loadTarget(t, "./python_wheel/python_wheel", "default") - diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + ctx := context.Background() + diags := bundle.Apply(ctx, b, phases.Build()) require.NoError(t, diags.Error()) matches, err := filepath.Glob("./python_wheel/python_wheel/my_test_code/dist/my_test_code-*.whl") @@ -32,11 +31,10 @@ func TestPythonWheelBuild(t *testing.T) { } func TestPythonWheelBuildAutoDetect(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_artifact") - require.NoError(t, err) + b := loadTarget(t, "./python_wheel/python_wheel_no_artifact", "default") - diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + ctx := context.Background() + diags := bundle.Apply(ctx, b, phases.Build()) require.NoError(t, diags.Error()) matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact/dist/my_test_code-*.whl") @@ -49,11 +47,10 @@ func TestPythonWheelBuildAutoDetect(t *testing.T) { } func TestPythonWheelBuildAutoDetectWithNotebookTask(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_artifact_notebook") - require.NoError(t, err) + b := loadTarget(t, "./python_wheel/python_wheel_no_artifact_notebook", "default") - diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + ctx := context.Background() + diags := bundle.Apply(ctx, b, phases.Build()) require.NoError(t, diags.Error()) matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact_notebook/dist/my_test_code-*.whl") @@ -66,11 +63,10 @@ func TestPythonWheelBuildAutoDetectWithNotebookTask(t *testing.T) { } func TestPythonWheelWithDBFSLib(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./python_wheel/python_wheel_dbfs_lib") - require.NoError(t, err) + b := loadTarget(t, "./python_wheel/python_wheel_dbfs_lib", "default") - diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + ctx := context.Background() + diags := bundle.Apply(ctx, b, phases.Build()) require.NoError(t, diags.Error()) match := libraries.ExpandGlobReferences() @@ -79,11 +75,11 @@ func TestPythonWheelWithDBFSLib(t *testing.T) { } func TestPythonWheelBuildNoBuildJustUpload(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_artifact_no_setup") - require.NoError(t, err) + b := loadTarget(t, "./python_wheel/python_wheel_no_artifact_no_setup", "default") - b.Config.Workspace.ArtifactPath = "/foo/bar" + ctx := context.Background() + diags := bundle.Apply(ctx, b, phases.Build()) + require.NoError(t, diags.Error()) mockFiler := mockfiler.NewMockFiler(t) mockFiler.EXPECT().Write( @@ -94,20 +90,20 @@ func TestPythonWheelBuildNoBuildJustUpload(t *testing.T) { filer.CreateParentDirectories, ).Return(nil) - u := libraries.UploadWithClient(mockFiler) - diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build(), libraries.ExpandGlobReferences(), u)) + diags = bundle.Apply(ctx, b, bundle.Seq( + libraries.ExpandGlobReferences(), + libraries.UploadWithClient(mockFiler), + )) require.NoError(t, diags.Error()) require.Empty(t, diags) - require.Equal(t, "/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[0].Libraries[0].Whl) } func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./python_wheel/environment_key") - require.NoError(t, err) + b := loadTarget(t, "./python_wheel/environment_key", "default") - diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + ctx := context.Background() + diags := bundle.Apply(ctx, b, phases.Build()) require.NoError(t, diags.Error()) matches, err := filepath.Glob("./python_wheel/environment_key/my_test_code/dist/my_test_code-*.whl") @@ -120,11 +116,10 @@ func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) { } func TestPythonWheelBuildMultiple(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./python_wheel/python_wheel_multiple") - require.NoError(t, err) + b := loadTarget(t, "./python_wheel/python_wheel_multiple", "default") - diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + ctx := context.Background() + diags := bundle.Apply(ctx, b, phases.Build()) require.NoError(t, diags.Error()) matches, err := filepath.Glob("./python_wheel/python_wheel_multiple/my_test_code/dist/my_test_code*.whl") @@ -137,11 +132,10 @@ func TestPythonWheelBuildMultiple(t *testing.T) { } func TestPythonWheelNoBuild(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_build") - require.NoError(t, err) + b := loadTarget(t, "./python_wheel/python_wheel_no_build", "default") - diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + ctx := context.Background() + diags := bundle.Apply(ctx, b, phases.Build()) require.NoError(t, diags.Error()) match := libraries.ExpandGlobReferences()