From 3e40a0c2f198e50d20aefb2ea39607147d416065 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Mon, 13 Jan 2025 13:19:12 +0100 Subject: [PATCH] Encourage the use of root_path in production to ensure single deployment (#1712) ## Changes This updates `mode: production` to allow `root_path` to indicate uniqueness. Historically, we required `run_as` for this, which isn't actually very effective for that purpose. `run_as` also had the problem that it doesn't work for pipelines. This is a cherry-pick from https://github.com/databricks/cli/pull/1387 --------- Co-authored-by: Pieter Noordhuis --- bundle/bundle.go | 3 +++ bundle/config/mutator/process_target_mode.go | 22 +++++++++++++++++-- .../mutator/process_target_mode_test.go | 21 ++++++++++++++++-- bundle/config/mutator/select_target.go | 7 ++++-- bundle/config/root.go | 4 ++-- libs/diag/diagnostic.go | 10 +++++++++ 6 files changed, 59 insertions(+), 8 deletions(-) diff --git a/bundle/bundle.go b/bundle/bundle.go index 1f5e2a294..3bf4ffb62 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -57,6 +57,9 @@ type Bundle struct { // It is loaded from the bundle configuration files and mutators may update it. Config config.Root + // Target stores a snapshot of the Root.Bundle.Target configuration when it was selected by SelectTarget. + Target *config.Target `json:"target_config,omitempty" bundle:"internal"` + // Metadata about the bundle deployment. This is the interface Databricks services // rely on to integrate with bundles when they need additional information about // a bundle deployment. diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go index 44b53681d..0fe6bd54f 100644 --- a/bundle/config/mutator/process_target_mode.go +++ b/bundle/config/mutator/process_target_mode.go @@ -2,6 +2,7 @@ package mutator import ( "context" + "fmt" "strings" "github.com/databricks/cli/bundle" @@ -146,8 +147,21 @@ func validateProductionMode(ctx context.Context, b *bundle.Bundle, isPrincipalUs } } - if !isPrincipalUsed && !isRunAsSet(r) { - return diag.Errorf("'run_as' must be set for all jobs when using 'mode: production'") + // We need to verify that there is only a single deployment of the current target. + // The best way to enforce this is to explicitly set root_path. + advice := fmt.Sprintf( + "set 'workspace.root_path' to make sure only one copy is deployed. A common practice is to use a username or principal name in this path, i.e. root_path: /Workspace/Users/%s/.bundle/${bundle.name}/${bundle.target}", + b.Config.Workspace.CurrentUser.UserName, + ) + if !isExplicitRootSet(b) { + if isRunAsSet(r) || isPrincipalUsed { + // Just setting run_as is not enough to guarantee a single deployment, + // and neither is setting a principal. + // We only show a warning for these cases since we didn't historically + // report an error for them. + return diag.Recommendationf("target with 'mode: production' should %s", advice) + } + return diag.Errorf("target with 'mode: production' must %s", advice) } return nil } @@ -164,6 +178,10 @@ func isRunAsSet(r config.Resources) bool { return true } +func isExplicitRootSet(b *bundle.Bundle) bool { + return b.Target != nil && b.Target.Workspace != nil && b.Target.Workspace.RootPath != "" +} + func (m *processTargetMode) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { switch b.Config.Bundle.Mode { case config.Development: diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index 097c092a6..6df88d067 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -321,7 +321,7 @@ func TestProcessTargetModeProduction(t *testing.T) { b := mockBundle(config.Production) diags := validateProductionMode(context.Background(), b, false) - require.ErrorContains(t, diags.Error(), "run_as") + require.ErrorContains(t, diags.Error(), "target with 'mode: production' must set 'workspace.root_path' to make sure only one copy is deployed. A common practice is to use a username or principal name in this path, i.e. root_path: /Workspace/Users/lennart@company.com/.bundle/${bundle.name}/${bundle.target}") b.Config.Workspace.StatePath = "/Shared/.bundle/x/y/state" b.Config.Workspace.ArtifactPath = "/Shared/.bundle/x/y/artifacts" @@ -329,7 +329,7 @@ func TestProcessTargetModeProduction(t *testing.T) { b.Config.Workspace.ResourcePath = "/Shared/.bundle/x/y/resources" diags = validateProductionMode(context.Background(), b, false) - require.ErrorContains(t, diags.Error(), "production") + require.ErrorContains(t, diags.Error(), "target with 'mode: production' must set 'workspace.root_path' to make sure only one copy is deployed. A common practice is to use a username or principal name in this path, i.e. root_path: /Workspace/Users/lennart@company.com/.bundle/${bundle.name}/${bundle.target}") permissions := []resources.Permission{ { @@ -375,6 +375,23 @@ func TestProcessTargetModeProductionOkForPrincipal(t *testing.T) { require.NoError(t, diags.Error()) } +func TestProcessTargetModeProductionOkWithRootPath(t *testing.T) { + b := mockBundle(config.Production) + + // Our target has all kinds of problems when not using service principals ... + diags := validateProductionMode(context.Background(), b, false) + require.Error(t, diags.Error()) + + // ... but we're okay if we specify a root path + b.Target = &config.Target{ + Workspace: &config.Workspace{ + RootPath: "some-root-path", + }, + } + diags = validateProductionMode(context.Background(), b, false) + require.NoError(t, diags.Error()) +} + // Make sure that we have test coverage for all resource types func TestAllResourcesMocked(t *testing.T) { b := mockBundle(config.Development) diff --git a/bundle/config/mutator/select_target.go b/bundle/config/mutator/select_target.go index 178686b6e..ce18da4f5 100644 --- a/bundle/config/mutator/select_target.go +++ b/bundle/config/mutator/select_target.go @@ -15,6 +15,7 @@ type selectTarget struct { } // SelectTarget merges the specified target into the root configuration. +// After merging, it removes the 'Targets' section from the configuration. func SelectTarget(name string) bundle.Mutator { return &selectTarget{ name: name, @@ -31,7 +32,7 @@ func (m *selectTarget) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnosti } // Get specified target - _, ok := b.Config.Targets[m.name] + target, ok := b.Config.Targets[m.name] if !ok { return diag.Errorf("%s: no such target. Available targets: %s", m.name, strings.Join(maps.Keys(b.Config.Targets), ", ")) } @@ -43,13 +44,15 @@ func (m *selectTarget) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnosti } // Store specified target in configuration for reference. + b.Target = target b.Config.Bundle.Target = m.name // We do this for backward compatibility. // TODO: remove when Environments section is not supported anymore. b.Config.Bundle.Environment = b.Config.Bundle.Target - // Clear targets after loading. + // Cleanup the original targets and environments sections since they + // show up in the JSON output of the 'summary' and 'validate' commands. b.Config.Targets = nil b.Config.Environments = nil diff --git a/bundle/config/root.go b/bundle/config/root.go index 91c15fd9d..21804110a 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -47,8 +47,8 @@ type Root struct { // Targets can be used to differentiate settings and resources between // bundle deployment targets (e.g. development, staging, production). - // If not specified, the code below initializes this field with a - // single default-initialized target called "default". + // Note that this field is set to 'nil' by the SelectTarget mutator; + // use bundle.Bundle.Target to access the selected target configuration. Targets map[string]*Target `json:"targets,omitempty"` // DEPRECATED. Left for backward compatibility with Targets diff --git a/libs/diag/diagnostic.go b/libs/diag/diagnostic.go index a4f8c7b6b..0c7699b4e 100644 --- a/libs/diag/diagnostic.go +++ b/libs/diag/diagnostic.go @@ -86,6 +86,16 @@ func Infof(format string, args ...any) Diagnostics { } } +// Recommendationf creates a new recommendation diagnostic. +func Recommendationf(format string, args ...any) Diagnostics { + return []Diagnostic{ + { + Severity: Recommendation, + Summary: fmt.Sprintf(format, args...), + }, + } +} + // Diagnostics holds zero or more instances of [Diagnostic]. type Diagnostics []Diagnostic