From 044a00c7f9fc2f2d35794062b63a54d01081ffa4 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 2 Oct 2024 15:53:24 +0200 Subject: [PATCH 1/3] Add an error if state files grow bigger than the export limit (#1795) ## Changes Currently API limits on exporting files from workspaces are set at 10 MBs while uploading to is 500 MBs. We want to prevent users running into deadlock when they won't be able to pull state file anymore so we prevent from uploading large state files (over 10 MBs) to Databricks workspace. --- bundle/deploy/state_push.go | 13 +++++++++++ bundle/deploy/terraform/state_push.go | 11 +++++++++ bundle/deploy/terraform/state_push_test.go | 27 ++++++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/bundle/deploy/state_push.go b/bundle/deploy/state_push.go index 176a907c..6912414c 100644 --- a/bundle/deploy/state_push.go +++ b/bundle/deploy/state_push.go @@ -10,6 +10,8 @@ import ( "github.com/databricks/cli/libs/log" ) +const MaxStateFileSize = 10 * 1024 * 1024 // 10MB + type statePush struct { filerFactory FilerFactory } @@ -35,6 +37,17 @@ func (s *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic } defer local.Close() + if !b.Config.Bundle.Force { + state, err := local.Stat() + if err != nil { + return diag.FromErr(err) + } + + if state.Size() > MaxStateFileSize { + return diag.Errorf("Deployment state file size exceeds the maximum allowed size of %d bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag.", MaxStateFileSize) + } + } + log.Infof(ctx, "Writing local deployment state file to remote state directory") err = f.Write(ctx, DeploymentStateFileName, local, filer.CreateParentDirectories, filer.OverwriteIfExists) if err != nil { diff --git a/bundle/deploy/terraform/state_push.go b/bundle/deploy/terraform/state_push.go index 6cdde137..84d8e767 100644 --- a/bundle/deploy/terraform/state_push.go +++ b/bundle/deploy/terraform/state_push.go @@ -47,6 +47,17 @@ func (l *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic } defer local.Close() + if !b.Config.Bundle.Force { + state, err := local.Stat() + if err != nil { + return diag.FromErr(err) + } + + if state.Size() > deploy.MaxStateFileSize { + return diag.Errorf("Terraform state file size exceeds the maximum allowed size of %d bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag", deploy.MaxStateFileSize) + } + } + // Upload state file from local cache directory to filer. cmdio.LogString(ctx, "Updating deployment state...") log.Infof(ctx, "Writing local state file to remote state directory") diff --git a/bundle/deploy/terraform/state_push_test.go b/bundle/deploy/terraform/state_push_test.go index e022dee1..4cc52b7a 100644 --- a/bundle/deploy/terraform/state_push_test.go +++ b/bundle/deploy/terraform/state_push_test.go @@ -3,6 +3,7 @@ package terraform import ( "context" "encoding/json" + "fmt" "io" "testing" @@ -59,3 +60,29 @@ func TestStatePush(t *testing.T) { diags := bundle.Apply(ctx, b, m) assert.NoError(t, diags.Error()) } + +func TestStatePushLargeState(t *testing.T) { + mock := mockfiler.NewMockFiler(t) + m := &statePush{ + identityFiler(mock), + } + + ctx := context.Background() + b := statePushTestBundle(t) + + largeState := map[string]any{} + for i := 0; i < 1000000; i++ { + largeState[fmt.Sprintf("field_%d", i)] = i + } + + // Write a stale local state file. + writeLocalState(t, ctx, b, largeState) + diags := bundle.Apply(ctx, b, m) + assert.ErrorContains(t, diags.Error(), "Terraform state file size exceeds the maximum allowed size of 10485760 bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag") + + // Force the write. + b = statePushTestBundle(t) + b.Config.Bundle.Force = true + diags = bundle.Apply(ctx, b, m) + assert.NoError(t, diags.Error()) +} From 80d55f454066a9c1666cb53387f9e735494fe72a Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 2 Oct 2024 15:55:40 +0200 Subject: [PATCH 2/3] Add resource path field to bundle workspace configuration (#1800) ## Changes Default workspace path for resources with a presence in the workspace tree. Note: this path is **not** created automatically (yet). We need this only for dashboards (so far), so can take care of creation if one or more dashboards are part of a deployment. This saves an API call for deployments where this is not necessary. ## Tests Expanded existing tests. --- bundle/config/mutator/default_workspace_paths.go | 4 ++++ bundle/config/mutator/default_workspace_paths_test.go | 3 +++ bundle/config/mutator/process_target_mode.go | 9 ++++++--- bundle/config/workspace.go | 5 +++++ 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/bundle/config/mutator/default_workspace_paths.go b/bundle/config/mutator/default_workspace_paths.go index 71e562b5..02a1ddb3 100644 --- a/bundle/config/mutator/default_workspace_paths.go +++ b/bundle/config/mutator/default_workspace_paths.go @@ -29,6 +29,10 @@ func (m *defineDefaultWorkspacePaths) Apply(ctx context.Context, b *bundle.Bundl b.Config.Workspace.FilePath = path.Join(root, "files") } + if b.Config.Workspace.ResourcePath == "" { + b.Config.Workspace.ResourcePath = path.Join(root, "resources") + } + if b.Config.Workspace.ArtifactPath == "" { b.Config.Workspace.ArtifactPath = path.Join(root, "artifacts") } diff --git a/bundle/config/mutator/default_workspace_paths_test.go b/bundle/config/mutator/default_workspace_paths_test.go index 0ba20ea2..6779c373 100644 --- a/bundle/config/mutator/default_workspace_paths_test.go +++ b/bundle/config/mutator/default_workspace_paths_test.go @@ -22,6 +22,7 @@ func TestDefineDefaultWorkspacePaths(t *testing.T) { diags := bundle.Apply(context.Background(), b, mutator.DefineDefaultWorkspacePaths()) require.NoError(t, diags.Error()) assert.Equal(t, "/files", b.Config.Workspace.FilePath) + assert.Equal(t, "/resources", b.Config.Workspace.ResourcePath) assert.Equal(t, "/artifacts", b.Config.Workspace.ArtifactPath) assert.Equal(t, "/state", b.Config.Workspace.StatePath) } @@ -32,6 +33,7 @@ func TestDefineDefaultWorkspacePathsAlreadySet(t *testing.T) { Workspace: config.Workspace{ RootPath: "/", FilePath: "/foo/bar", + ResourcePath: "/foo/bar", ArtifactPath: "/foo/bar", StatePath: "/foo/bar", }, @@ -40,6 +42,7 @@ func TestDefineDefaultWorkspacePathsAlreadySet(t *testing.T) { diags := bundle.Apply(context.Background(), b, mutator.DefineDefaultWorkspacePaths()) require.NoError(t, diags.Error()) assert.Equal(t, "/foo/bar", b.Config.Workspace.FilePath) + assert.Equal(t, "/foo/bar", b.Config.Workspace.ResourcePath) assert.Equal(t, "/foo/bar", b.Config.Workspace.ArtifactPath) assert.Equal(t, "/foo/bar", b.Config.Workspace.StatePath) } diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go index 70382f05..9944f6ff 100644 --- a/bundle/config/mutator/process_target_mode.go +++ b/bundle/config/mutator/process_target_mode.go @@ -118,15 +118,18 @@ func findNonUserPath(b *bundle.Bundle) string { if b.Config.Workspace.RootPath != "" && !containsName(b.Config.Workspace.RootPath) { return "root_path" } - if b.Config.Workspace.StatePath != "" && !containsName(b.Config.Workspace.StatePath) { - return "state_path" - } if b.Config.Workspace.FilePath != "" && !containsName(b.Config.Workspace.FilePath) { return "file_path" } + if b.Config.Workspace.ResourcePath != "" && !containsName(b.Config.Workspace.ResourcePath) { + return "resource_path" + } if b.Config.Workspace.ArtifactPath != "" && !containsName(b.Config.Workspace.ArtifactPath) { return "artifact_path" } + if b.Config.Workspace.StatePath != "" && !containsName(b.Config.Workspace.StatePath) { + return "state_path" + } return "" } diff --git a/bundle/config/workspace.go b/bundle/config/workspace.go index efc5caa6..878d0783 100644 --- a/bundle/config/workspace.go +++ b/bundle/config/workspace.go @@ -54,6 +54,11 @@ type Workspace struct { // This defaults to "${workspace.root}/files". FilePath string `json:"file_path,omitempty"` + // Remote workspace path for resources with a presence in the workspace. + // These are kept outside [FilePath] to avoid potential naming collisions. + // This defaults to "${workspace.root}/resources". + ResourcePath string `json:"resource_path,omitempty"` + // Remote workspace path for build artifacts. // This defaults to "${workspace.root}/artifacts". ArtifactPath string `json:"artifact_path,omitempty"` From a8cff48c0b4d747cb769d09724fecdf3c5a3f25b Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 2 Oct 2024 17:34:00 +0200 Subject: [PATCH 3/3] Always prepend bundle remote paths with /Workspace (#1724) ## Changes Due to platform changes, all libraries, notebooks and etc. paths used in Databricks must be started with either /Workspace or /Volumes prefix. This PR makes sure that all bundle paths are correctly prefixed. Note: this change is a breaking change if user previously configured and used `/Workspace/Workspace` folder in their workspace file system or having `/Workspace/${workspace.root_path}...` pattern configured anywhere in their bundle config Fixes: #1751 AI: - [x] Scan DABs config and error out on `/Workspace/${workspace.root_path}...` pattern usage ## Tests Added unit tests --------- Co-authored-by: Pieter Noordhuis --- .../config/mutator/expand_workspace_root.go | 2 +- .../mutator/expand_workspace_root_test.go | 2 +- .../mutator/prepend_workspace_prefix.go | 67 +++++++++++++++ .../mutator/prepend_workspace_prefix_test.go | 79 +++++++++++++++++ .../mutator/rewrite_workspace_prefix.go | 72 ++++++++++++++++ .../mutator/rewrite_workspace_prefix_test.go | 85 +++++++++++++++++++ bundle/config/workspace.go | 2 +- bundle/phases/initialize.go | 7 ++ bundle/tests/pipeline_glob_paths_test.go | 2 +- .../tests/relative_path_translation_test.go | 8 +- internal/bundle/deploy_test.go | 2 +- internal/bundle/helpers.go | 2 +- .../{{.project_name}}/databricks.yml.tmpl | 4 +- .../{{.project_name}}/databricks.yml.tmpl | 4 +- .../{{.project_name}}.pipeline.yml.tmpl | 2 +- .../{{.project_name}}/databricks.yml.tmpl | 4 +- 16 files changed, 327 insertions(+), 17 deletions(-) create mode 100644 bundle/config/mutator/prepend_workspace_prefix.go create mode 100644 bundle/config/mutator/prepend_workspace_prefix_test.go create mode 100644 bundle/config/mutator/rewrite_workspace_prefix.go create mode 100644 bundle/config/mutator/rewrite_workspace_prefix_test.go diff --git a/bundle/config/mutator/expand_workspace_root.go b/bundle/config/mutator/expand_workspace_root.go index 8954abd4..3f0547de 100644 --- a/bundle/config/mutator/expand_workspace_root.go +++ b/bundle/config/mutator/expand_workspace_root.go @@ -33,7 +33,7 @@ func (m *expandWorkspaceRoot) Apply(ctx context.Context, b *bundle.Bundle) diag. } if strings.HasPrefix(root, "~/") { - home := fmt.Sprintf("/Users/%s", currentUser.UserName) + home := fmt.Sprintf("/Workspace/Users/%s", currentUser.UserName) b.Config.Workspace.RootPath = path.Join(home, root[2:]) } diff --git a/bundle/config/mutator/expand_workspace_root_test.go b/bundle/config/mutator/expand_workspace_root_test.go index e6260dbd..40bf35ca 100644 --- a/bundle/config/mutator/expand_workspace_root_test.go +++ b/bundle/config/mutator/expand_workspace_root_test.go @@ -27,7 +27,7 @@ func TestExpandWorkspaceRoot(t *testing.T) { } diags := bundle.Apply(context.Background(), b, mutator.ExpandWorkspaceRoot()) require.NoError(t, diags.Error()) - assert.Equal(t, "/Users/jane@doe.com/foo", b.Config.Workspace.RootPath) + assert.Equal(t, "/Workspace/Users/jane@doe.com/foo", b.Config.Workspace.RootPath) } func TestExpandWorkspaceRootDoesNothing(t *testing.T) { diff --git a/bundle/config/mutator/prepend_workspace_prefix.go b/bundle/config/mutator/prepend_workspace_prefix.go new file mode 100644 index 00000000..dd467344 --- /dev/null +++ b/bundle/config/mutator/prepend_workspace_prefix.go @@ -0,0 +1,67 @@ +package mutator + +import ( + "context" + "fmt" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type prependWorkspacePrefix struct{} + +// PrependWorkspacePrefix prepends the workspace root path to all paths in the bundle. +func PrependWorkspacePrefix() bundle.Mutator { + return &prependWorkspacePrefix{} +} + +func (m *prependWorkspacePrefix) Name() string { + return "PrependWorkspacePrefix" +} + +var skipPrefixes = []string{ + "/Workspace/", + "/Volumes/", +} + +func (m *prependWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + patterns := []dyn.Pattern{ + dyn.NewPattern(dyn.Key("workspace"), dyn.Key("root_path")), + dyn.NewPattern(dyn.Key("workspace"), dyn.Key("file_path")), + dyn.NewPattern(dyn.Key("workspace"), dyn.Key("artifact_path")), + dyn.NewPattern(dyn.Key("workspace"), dyn.Key("state_path")), + } + + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + var err error + for _, pattern := range patterns { + v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, pv dyn.Value) (dyn.Value, error) { + path, ok := pv.AsString() + if !ok { + return dyn.InvalidValue, fmt.Errorf("expected string, got %s", v.Kind()) + } + + for _, prefix := range skipPrefixes { + if strings.HasPrefix(path, prefix) { + return pv, nil + } + } + + return dyn.NewValue(fmt.Sprintf("/Workspace%s", path), v.Locations()), nil + }) + + if err != nil { + return dyn.InvalidValue, err + } + } + return v, nil + }) + + if err != nil { + return diag.FromErr(err) + } + + return nil +} diff --git a/bundle/config/mutator/prepend_workspace_prefix_test.go b/bundle/config/mutator/prepend_workspace_prefix_test.go new file mode 100644 index 00000000..287c694d --- /dev/null +++ b/bundle/config/mutator/prepend_workspace_prefix_test.go @@ -0,0 +1,79 @@ +package mutator + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/stretchr/testify/require" +) + +func TestPrependWorkspacePrefix(t *testing.T) { + testCases := []struct { + path string + expected string + }{ + { + path: "/Users/test", + expected: "/Workspace/Users/test", + }, + { + path: "/Shared/test", + expected: "/Workspace/Shared/test", + }, + { + path: "/Workspace/Users/test", + expected: "/Workspace/Users/test", + }, + { + path: "/Volumes/Users/test", + expected: "/Volumes/Users/test", + }, + } + + for _, tc := range testCases { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + RootPath: tc.path, + ArtifactPath: tc.path, + FilePath: tc.path, + StatePath: tc.path, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, PrependWorkspacePrefix()) + require.Empty(t, diags) + require.Equal(t, tc.expected, b.Config.Workspace.RootPath) + require.Equal(t, tc.expected, b.Config.Workspace.ArtifactPath) + require.Equal(t, tc.expected, b.Config.Workspace.FilePath) + require.Equal(t, tc.expected, b.Config.Workspace.StatePath) + } +} + +func TestPrependWorkspaceForDefaultConfig(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Bundle: config.Bundle{ + Name: "test", + Target: "dev", + }, + Workspace: config.Workspace{ + CurrentUser: &config.User{ + User: &iam.User{ + UserName: "jane@doe.com", + }, + }, + }, + }, + } + diags := bundle.Apply(context.Background(), b, bundle.Seq(DefineDefaultWorkspaceRoot(), ExpandWorkspaceRoot(), DefineDefaultWorkspacePaths(), PrependWorkspacePrefix())) + require.Empty(t, diags) + require.Equal(t, "/Workspace/Users/jane@doe.com/.bundle/test/dev", b.Config.Workspace.RootPath) + require.Equal(t, "/Workspace/Users/jane@doe.com/.bundle/test/dev/artifacts", b.Config.Workspace.ArtifactPath) + require.Equal(t, "/Workspace/Users/jane@doe.com/.bundle/test/dev/files", b.Config.Workspace.FilePath) + require.Equal(t, "/Workspace/Users/jane@doe.com/.bundle/test/dev/state", b.Config.Workspace.StatePath) +} diff --git a/bundle/config/mutator/rewrite_workspace_prefix.go b/bundle/config/mutator/rewrite_workspace_prefix.go new file mode 100644 index 00000000..8a39ee8a --- /dev/null +++ b/bundle/config/mutator/rewrite_workspace_prefix.go @@ -0,0 +1,72 @@ +package mutator + +import ( + "context" + "fmt" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type rewriteWorkspacePrefix struct{} + +// RewriteWorkspacePrefix finds any strings in bundle configration that have +// workspace prefix plus workspace path variable used and removes workspace prefix from it. +func RewriteWorkspacePrefix() bundle.Mutator { + return &rewriteWorkspacePrefix{} +} + +func (m *rewriteWorkspacePrefix) Name() string { + return "RewriteWorkspacePrefix" +} + +func (m *rewriteWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + diags := diag.Diagnostics{} + paths := map[string]string{ + "/Workspace/${workspace.root_path}": "${workspace.root_path}", + "/Workspace${workspace.root_path}": "${workspace.root_path}", + "/Workspace/${workspace.file_path}": "${workspace.file_path}", + "/Workspace${workspace.file_path}": "${workspace.file_path}", + "/Workspace/${workspace.artifact_path}": "${workspace.artifact_path}", + "/Workspace${workspace.artifact_path}": "${workspace.artifact_path}", + "/Workspace/${workspace.state_path}": "${workspace.state_path}", + "/Workspace${workspace.state_path}": "${workspace.state_path}", + } + + err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { + // Walk through the bundle configuration, check all the string leafs and + // see if any of the prefixes are used in the remote path. + return dyn.Walk(root, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + vv, ok := v.AsString() + if !ok { + return v, nil + } + + for path, replacePath := range paths { + if strings.Contains(vv, path) { + newPath := strings.Replace(vv, path, replacePath, 1) + diags = append(diags, diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("substring %q found in %q. Please update this to %q.", path, vv, newPath), + Detail: "For more information, please refer to: https://docs.databricks.com/en/release-notes/dev-tools/bundles.html#workspace-paths", + Locations: v.Locations(), + Paths: []dyn.Path{p}, + }) + + // Remove the workspace prefix from the string. + return dyn.NewValue(newPath, v.Locations()), nil + } + } + + return v, nil + }) + }) + + if err != nil { + return diag.FromErr(err) + } + + return diags +} diff --git a/bundle/config/mutator/rewrite_workspace_prefix_test.go b/bundle/config/mutator/rewrite_workspace_prefix_test.go new file mode 100644 index 00000000..d75ec89d --- /dev/null +++ b/bundle/config/mutator/rewrite_workspace_prefix_test.go @@ -0,0 +1,85 @@ +package mutator + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/require" +) + +func TestNoWorkspacePrefixUsed(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + RootPath: "/Workspace/Users/test", + ArtifactPath: "/Workspace/Users/test/artifacts", + FilePath: "/Workspace/Users/test/files", + StatePath: "/Workspace/Users/test/state", + }, + + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "test_job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + SparkPythonTask: &jobs.SparkPythonTask{ + PythonFile: "/Workspace/${workspace.root_path}/file1.py", + }, + }, + { + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "/Workspace${workspace.file_path}/notebook1", + }, + Libraries: []compute.Library{ + { + Jar: "/Workspace/${workspace.artifact_path}/jar1.jar", + }, + }, + }, + { + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "${workspace.file_path}/notebook2", + }, + Libraries: []compute.Library{ + { + Jar: "${workspace.artifact_path}/jar2.jar", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, RewriteWorkspacePrefix()) + require.Len(t, diags, 3) + + expectedErrors := map[string]bool{ + `substring "/Workspace/${workspace.root_path}" found in "/Workspace/${workspace.root_path}/file1.py". Please update this to "${workspace.root_path}/file1.py".`: true, + `substring "/Workspace${workspace.file_path}" found in "/Workspace${workspace.file_path}/notebook1". Please update this to "${workspace.file_path}/notebook1".`: true, + `substring "/Workspace/${workspace.artifact_path}" found in "/Workspace/${workspace.artifact_path}/jar1.jar". Please update this to "${workspace.artifact_path}/jar1.jar".`: true, + } + + for _, d := range diags { + require.Equal(t, d.Severity, diag.Warning) + require.Contains(t, expectedErrors, d.Summary) + delete(expectedErrors, d.Summary) + } + + require.Equal(t, "${workspace.root_path}/file1.py", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[0].SparkPythonTask.PythonFile) + require.Equal(t, "${workspace.file_path}/notebook1", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[1].NotebookTask.NotebookPath) + require.Equal(t, "${workspace.artifact_path}/jar1.jar", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[1].Libraries[0].Jar) + require.Equal(t, "${workspace.file_path}/notebook2", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[2].NotebookTask.NotebookPath) + require.Equal(t, "${workspace.artifact_path}/jar2.jar", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[2].Libraries[0].Jar) + +} diff --git a/bundle/config/workspace.go b/bundle/config/workspace.go index 878d0783..e64808ab 100644 --- a/bundle/config/workspace.go +++ b/bundle/config/workspace.go @@ -47,7 +47,7 @@ type Workspace struct { // Remote workspace base path for deployment state, for artifacts, as synchronization target. // This defaults to "~/.bundle/${bundle.name}/${bundle.target}" where "~" expands to - // the current user's home directory in the workspace (e.g. `/Users/jane@doe.com`). + // the current user's home directory in the workspace (e.g. `/Workspace/Users/jane@doe.com`). RootPath string `json:"root_path,omitempty"` // Remote workspace path to synchronize local files to. diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 93ce61b2..a41819c7 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -39,9 +39,16 @@ func Initialize() bundle.Mutator { mutator.MergePipelineClusters(), mutator.InitializeWorkspaceClient(), mutator.PopulateCurrentUser(), + mutator.DefineDefaultWorkspaceRoot(), mutator.ExpandWorkspaceRoot(), mutator.DefineDefaultWorkspacePaths(), + mutator.PrependWorkspacePrefix(), + + // This mutator needs to be run before variable interpolation because it + // searches for strings with variable references in them. + mutator.RewriteWorkspacePrefix(), + mutator.SetVariables(), // Intentionally placed before ResolveVariableReferencesInLookup, ResolveResourceReferences, // ResolveVariableReferencesInComplexVariables and ResolveVariableReferences. diff --git a/bundle/tests/pipeline_glob_paths_test.go b/bundle/tests/pipeline_glob_paths_test.go index c1c62cfb..d17f0404 100644 --- a/bundle/tests/pipeline_glob_paths_test.go +++ b/bundle/tests/pipeline_glob_paths_test.go @@ -11,7 +11,7 @@ func TestExpandPipelineGlobPaths(t *testing.T) { require.NoError(t, diags.Error()) require.Equal( t, - "/Users/user@domain.com/.bundle/pipeline_glob_paths/default/files/dlt/nyc_taxi_loader", + "/Workspace/Users/user@domain.com/.bundle/pipeline_glob_paths/default/files/dlt/nyc_taxi_loader", b.Config.Resources.Pipelines["nyc_taxi_pipeline"].Libraries[0].Notebook.Path, ) } diff --git a/bundle/tests/relative_path_translation_test.go b/bundle/tests/relative_path_translation_test.go index 199871d2..0f553ac3 100644 --- a/bundle/tests/relative_path_translation_test.go +++ b/bundle/tests/relative_path_translation_test.go @@ -12,9 +12,9 @@ func TestRelativePathTranslationDefault(t *testing.T) { require.NoError(t, diags.Error()) t0 := b.Config.Resources.Jobs["job"].Tasks[0] - assert.Equal(t, "/remote/src/file1.py", t0.SparkPythonTask.PythonFile) + assert.Equal(t, "/Workspace/remote/src/file1.py", t0.SparkPythonTask.PythonFile) t1 := b.Config.Resources.Jobs["job"].Tasks[1] - assert.Equal(t, "/remote/src/file1.py", t1.SparkPythonTask.PythonFile) + assert.Equal(t, "/Workspace/remote/src/file1.py", t1.SparkPythonTask.PythonFile) } func TestRelativePathTranslationOverride(t *testing.T) { @@ -22,7 +22,7 @@ func TestRelativePathTranslationOverride(t *testing.T) { require.NoError(t, diags.Error()) t0 := b.Config.Resources.Jobs["job"].Tasks[0] - assert.Equal(t, "/remote/src/file2.py", t0.SparkPythonTask.PythonFile) + assert.Equal(t, "/Workspace/remote/src/file2.py", t0.SparkPythonTask.PythonFile) t1 := b.Config.Resources.Jobs["job"].Tasks[1] - assert.Equal(t, "/remote/src/file2.py", t1.SparkPythonTask.PythonFile) + assert.Equal(t, "/Workspace/remote/src/file2.py", t1.SparkPythonTask.PythonFile) } diff --git a/internal/bundle/deploy_test.go b/internal/bundle/deploy_test.go index 736c880d..88543585 100644 --- a/internal/bundle/deploy_test.go +++ b/internal/bundle/deploy_test.go @@ -236,7 +236,7 @@ func TestAccDeployBasicBundleLogs(t *testing.T) { stdout, stderr := blackBoxRun(t, root, "bundle", "deploy") assert.Equal(t, strings.Join([]string{ - fmt.Sprintf("Uploading bundle files to /Users/%s/.bundle/%s/files...", currentUser.UserName, uniqueId), + fmt.Sprintf("Uploading bundle files to /Workspace/Users/%s/.bundle/%s/files...", currentUser.UserName, uniqueId), "Deploying resources...", "Updating deployment state...", "Deployment complete!\n", diff --git a/internal/bundle/helpers.go b/internal/bundle/helpers.go index 3547c175..b8c81a8d 100644 --- a/internal/bundle/helpers.go +++ b/internal/bundle/helpers.go @@ -114,7 +114,7 @@ func getBundleRemoteRootPath(w *databricks.WorkspaceClient, t *testing.T, unique // Compute root path for the bundle deployment me, err := w.CurrentUser.Me(context.Background()) require.NoError(t, err) - root := fmt.Sprintf("/Users/%s/.bundle/%s", me.UserName, uniqueId) + root := fmt.Sprintf("/Workspace/Users/%s/.bundle/%s", me.UserName, uniqueId) return root } diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl index f96ce4fe..5594749a 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl @@ -24,8 +24,8 @@ targets: mode: production workspace: host: {{workspace_host}} - # We explicitly specify /Users/{{user_name}} to make sure we only have a single copy. - root_path: /Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} + # We explicitly specify /Workspace/Users/{{user_name}} to make sure we only have a single copy. + root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} permissions: - {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}} level: CAN_MANAGE diff --git a/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl index 8544dc83..c42b822a 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl @@ -21,8 +21,8 @@ targets: mode: production workspace: host: {{workspace_host}} - # We explicitly specify /Users/{{user_name}} to make sure we only have a single copy. - root_path: /Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} + # We explicitly specify /Workspace/Users/{{user_name}} to make sure we only have a single copy. + root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} permissions: - {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}} level: CAN_MANAGE diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl index bf469046..50e5ad97 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl @@ -15,4 +15,4 @@ resources: path: ../src/dlt_pipeline.ipynb configuration: - bundle.sourcePath: /Workspace/${workspace.file_path}/src + bundle.sourcePath: ${workspace.file_path}/src diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl index 55c1aae4..51d03e99 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl @@ -41,8 +41,8 @@ targets: mode: production workspace: host: {{workspace_host}} - # We explicitly specify /Users/{{user_name}} to make sure we only have a single copy. - root_path: /Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} + # We explicitly specify /Workspace/Users/{{user_name}} to make sure we only have a single copy. + root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} variables: warehouse_id: {{index ((regexp "[^/]+$").FindStringSubmatch .http_path) 0}} catalog: {{.default_catalog}}