From 6b22fdebbaa98d7190a0d62f2a24e17d34ee7d0e Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Tue, 5 Nov 2024 18:12:45 +0100 Subject: [PATCH] Prototype for in-place deployments in DEV mode --- bundle/config/mutator/apply_presets.go | 10 +++++++++ bundle/config/mutator/process_target_mode.go | 5 +++++ bundle/config/mutator/translate_paths.go | 23 ++++++++++++++++++++ bundle/config/presets.go | 5 +++++ bundle/deploy/files/upload.go | 6 +++++ 5 files changed, 49 insertions(+) diff --git a/bundle/config/mutator/apply_presets.go b/bundle/config/mutator/apply_presets.go index d2a1d0c7..4b29f005 100644 --- a/bundle/config/mutator/apply_presets.go +++ b/bundle/config/mutator/apply_presets.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -221,6 +222,15 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos dashboard.DisplayName = prefix + dashboard.DisplayName } + root := b.SyncRoot.Native() + _, ok := env.Lookup(ctx, envDatabricksRuntimeVersion) + isInWorkspace := ok && strings.HasPrefix(root, "/Workspace/") + + if !isInWorkspace { + disabled := false + b.Config.Presets.InPlaceDeployment = &disabled + } + return diags } diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go index 44b53681..6ea0e7ef 100644 --- a/bundle/config/mutator/process_target_mode.go +++ b/bundle/config/mutator/process_target_mode.go @@ -57,6 +57,11 @@ func transformDevelopmentMode(ctx context.Context, b *bundle.Bundle) { t.TriggerPauseStatus = config.Paused } + if !config.IsExplicitlyDisabled(t.InPlaceDeployment) { + enabled := true + t.InPlaceDeployment = &enabled + } + if !config.IsExplicitlyDisabled(t.PipelinesDevelopment) { enabled := true t.PipelinesDevelopment = &enabled diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 82b0b3ca..fe3f2f8b 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -11,6 +11,8 @@ import ( "strings" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/notebook" @@ -129,6 +131,9 @@ func (t *translateContext) translateNotebookPath(literal, localFullPath, localRe return "", ErrIsNotNotebook{localFullPath} } + if t.shouldTranslateRemotePaths(localFullPath) { + return localFullPath, nil + } // Upon import, notebooks are stripped of their extension. return strings.TrimSuffix(remotePath, filepath.Ext(localFullPath)), nil } @@ -144,6 +149,9 @@ func (t *translateContext) translateFilePath(literal, localFullPath, localRelPat if nb { return "", ErrIsNotebook{localFullPath} } + if t.shouldTranslateRemotePaths(localFullPath) { + return localFullPath, nil + } return remotePath, nil } @@ -155,10 +163,16 @@ func (t *translateContext) translateDirectoryPath(literal, localFullPath, localR if !info.IsDir() { return "", fmt.Errorf("%s is not a directory", localFullPath) } + if t.shouldTranslateRemotePaths(localFullPath) { + return localFullPath, nil + } return remotePath, nil } func (t *translateContext) translateNoOp(literal, localFullPath, localRelPath, remotePath string) (string, error) { + if t.shouldTranslateRemotePaths(localFullPath) { + return localFullPath, nil + } return localRelPath, nil } @@ -177,6 +191,10 @@ func (t *translateContext) retainLocalAbsoluteFilePath(literal, localFullPath, l } func (t *translateContext) translateNoOpWithPrefix(literal, localFullPath, localRelPath, remotePath string) (string, error) { + if t.shouldTranslateRemotePaths(localFullPath) { + return localFullPath, nil + } + if !strings.HasPrefix(localRelPath, ".") { localRelPath = "." + string(filepath.Separator) + localRelPath } @@ -217,6 +235,11 @@ func (t *translateContext) rewriteRelativeTo(p dyn.Path, v dyn.Value, fn rewrite return dyn.InvalidValue, err } +func (t *translateContext) shouldTranslateRemotePaths(localFullPath string) bool { + return config.IsExplicitlyEnabled(t.b.Config.Presets.InPlaceDeployment) && + libraries.IsWorkspacePath(localFullPath) +} + func (m *translatePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { t := &translateContext{ b: b, diff --git a/bundle/config/presets.go b/bundle/config/presets.go index 61009a25..c5fb6e10 100644 --- a/bundle/config/presets.go +++ b/bundle/config/presets.go @@ -17,6 +17,11 @@ type Presets struct { // JobsMaxConcurrentRuns is the default value for the max concurrent runs of jobs. JobsMaxConcurrentRuns int `json:"jobs_max_concurrent_runs,omitempty"` + // InPlaceDeployment indicates whether in-place deployment is enabled. Works only in workspace + // When set to true, resources created during deployment will point to source files in the workspace instead of their workspace copies. + // No resources will be uploaded to workspace + InPlaceDeployment *bool `json:"in_place_deployment,omitempty"` + // Tags to add to all resources. Tags map[string]string `json:"tags,omitempty"` } diff --git a/bundle/deploy/files/upload.go b/bundle/deploy/files/upload.go index bab4e176..bdad6f01 100644 --- a/bundle/deploy/files/upload.go +++ b/bundle/deploy/files/upload.go @@ -7,6 +7,7 @@ import ( "io/fs" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/permissions" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" @@ -23,6 +24,11 @@ func (m *upload) Name() string { } func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if config.IsExplicitlyEnabled(b.Config.Presets.InPlaceDeployment) { + cmdio.LogString(ctx, "Bundle files uploading skipped: in-place deployment is enabled") + return nil + } + cmdio.LogString(ctx, fmt.Sprintf("Uploading bundle files to %s...", b.Config.Workspace.FilePath)) opts, err := GetSyncOptions(ctx, bundle.ReadOnly(b)) if err != nil {