mirror of https://github.com/databricks/cli.git
fix: Move source-linked preset before variable resolution step
This commit is contained in:
parent
31ff82f381
commit
74f51d6f73
|
@ -9,7 +9,6 @@ import (
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/libs/dbr"
|
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/databricks/cli/libs/textutil"
|
"github.com/databricks/cli/libs/textutil"
|
||||||
|
@ -222,27 +221,6 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
|
||||||
dashboard.DisplayName = prefix + dashboard.DisplayName
|
dashboard.DisplayName = prefix + dashboard.DisplayName
|
||||||
}
|
}
|
||||||
|
|
||||||
if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) {
|
|
||||||
isDatabricksWorkspace := dbr.RunsOnRuntime(ctx) && strings.HasPrefix(b.SyncRootPath, "/Workspace/")
|
|
||||||
if !isDatabricksWorkspace {
|
|
||||||
target := b.Config.Bundle.Target
|
|
||||||
path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment"))
|
|
||||||
diags = diags.Append(
|
|
||||||
diag.Diagnostic{
|
|
||||||
Severity: diag.Warning,
|
|
||||||
Summary: "source-linked deployment is available only in the Databricks Workspace",
|
|
||||||
Paths: []dyn.Path{
|
|
||||||
path,
|
|
||||||
},
|
|
||||||
Locations: b.Config.GetLocations(path[2:].String()),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
disabled := false
|
|
||||||
b.Config.Presets.SourceLinkedDeployment = &disabled
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return diags
|
return diags
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -2,16 +2,12 @@ package mutator_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"runtime"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/mutator"
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
"github.com/databricks/cli/bundle/config/resources"
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
"github.com/databricks/cli/bundle/internal/bundletest"
|
|
||||||
"github.com/databricks/cli/libs/dbr"
|
|
||||||
"github.com/databricks/cli/libs/dyn"
|
|
||||||
"github.com/databricks/databricks-sdk-go/service/catalog"
|
"github.com/databricks/databricks-sdk-go/service/catalog"
|
||||||
"github.com/databricks/databricks-sdk-go/service/jobs"
|
"github.com/databricks/databricks-sdk-go/service/jobs"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
@ -398,87 +394,3 @@ func TestApplyPresetsResourceNotDefined(t *testing.T) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
|
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
|
|
||||||
}
|
|
||||||
|
|
||||||
testContext := context.Background()
|
|
||||||
enabled := true
|
|
||||||
disabled := false
|
|
||||||
workspacePath := "/Workspace/user.name@company.com"
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
bundlePath string
|
|
||||||
ctx context.Context
|
|
||||||
name string
|
|
||||||
initialValue *bool
|
|
||||||
expectedValue *bool
|
|
||||||
expectedWarning string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "preset enabled, bundle in Workspace, databricks runtime",
|
|
||||||
bundlePath: workspacePath,
|
|
||||||
ctx: dbr.MockRuntime(testContext, true),
|
|
||||||
initialValue: &enabled,
|
|
||||||
expectedValue: &enabled,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "preset enabled, bundle not in Workspace, databricks runtime",
|
|
||||||
bundlePath: "/Users/user.name@company.com",
|
|
||||||
ctx: dbr.MockRuntime(testContext, true),
|
|
||||||
initialValue: &enabled,
|
|
||||||
expectedValue: &disabled,
|
|
||||||
expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "preset enabled, bundle in Workspace, not databricks runtime",
|
|
||||||
bundlePath: workspacePath,
|
|
||||||
ctx: dbr.MockRuntime(testContext, false),
|
|
||||||
initialValue: &enabled,
|
|
||||||
expectedValue: &disabled,
|
|
||||||
expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "preset disabled, bundle in Workspace, databricks runtime",
|
|
||||||
bundlePath: workspacePath,
|
|
||||||
ctx: dbr.MockRuntime(testContext, true),
|
|
||||||
initialValue: &disabled,
|
|
||||||
expectedValue: &disabled,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "preset nil, bundle in Workspace, databricks runtime",
|
|
||||||
bundlePath: workspacePath,
|
|
||||||
ctx: dbr.MockRuntime(testContext, true),
|
|
||||||
initialValue: nil,
|
|
||||||
expectedValue: nil,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
SyncRootPath: tt.bundlePath,
|
|
||||||
Config: config.Root{
|
|
||||||
Presets: config.Presets{
|
|
||||||
SourceLinkedDeployment: tt.initialValue,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
bundletest.SetLocation(b, "presets.source_linked_deployment", []dyn.Location{{File: "databricks.yml"}})
|
|
||||||
diags := bundle.Apply(tt.ctx, b, mutator.ApplyPresets())
|
|
||||||
if diags.HasError() {
|
|
||||||
t.Fatalf("unexpected error: %v", diags)
|
|
||||||
}
|
|
||||||
|
|
||||||
if tt.expectedWarning != "" {
|
|
||||||
require.Equal(t, tt.expectedWarning, diags[0].Summary)
|
|
||||||
require.NotEmpty(t, diags[0].Locations)
|
|
||||||
}
|
|
||||||
|
|
||||||
require.Equal(t, tt.expectedValue, b.Config.Presets.SourceLinkedDeployment)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
@ -0,0 +1,60 @@
|
||||||
|
package mutator
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config"
|
||||||
|
"github.com/databricks/cli/libs/dbr"
|
||||||
|
"github.com/databricks/cli/libs/diag"
|
||||||
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
)
|
||||||
|
|
||||||
|
type applySourceLinkedDeploymentPreset struct{}
|
||||||
|
|
||||||
|
// Apply source-linked deployment preset
|
||||||
|
func ApplySourceLinkedDeploymentPreset() *applySourceLinkedDeploymentPreset {
|
||||||
|
return &applySourceLinkedDeploymentPreset{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *applySourceLinkedDeploymentPreset) Name() string {
|
||||||
|
return "ApplySourceLinkedDeploymentPreset"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||||
|
if config.IsExplicitlyDisabled(b.Config.Presets.SourceLinkedDeployment) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var diags diag.Diagnostics
|
||||||
|
isDatabricksWorkspace := dbr.RunsOnRuntime(ctx) && strings.HasPrefix(b.SyncRootPath, "/Workspace/")
|
||||||
|
|
||||||
|
if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) {
|
||||||
|
if !isDatabricksWorkspace {
|
||||||
|
target := b.Config.Bundle.Target
|
||||||
|
path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment"))
|
||||||
|
diags = diags.Append(
|
||||||
|
diag.Diagnostic{
|
||||||
|
Severity: diag.Warning,
|
||||||
|
Summary: "source-linked deployment is available only in the Databricks Workspace",
|
||||||
|
Paths: []dyn.Path{
|
||||||
|
path,
|
||||||
|
},
|
||||||
|
Locations: b.Config.GetLocations(path[2:].String()),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
disabled := false
|
||||||
|
b.Config.Presets.SourceLinkedDeployment = &disabled
|
||||||
|
return diags
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if isDatabricksWorkspace && b.Config.Bundle.Mode == config.Development {
|
||||||
|
enabled := true
|
||||||
|
b.Config.Presets.SourceLinkedDeployment = &enabled
|
||||||
|
}
|
||||||
|
|
||||||
|
return diags
|
||||||
|
}
|
|
@ -0,0 +1,116 @@
|
||||||
|
package mutator_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"runtime"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config"
|
||||||
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
|
"github.com/databricks/cli/libs/dbr"
|
||||||
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
|
||||||
|
}
|
||||||
|
|
||||||
|
testContext := context.Background()
|
||||||
|
enabled := true
|
||||||
|
disabled := false
|
||||||
|
workspacePath := "/Workspace/user.name@company.com"
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
bundlePath string
|
||||||
|
ctx context.Context
|
||||||
|
name string
|
||||||
|
mode config.Mode
|
||||||
|
initialValue *bool
|
||||||
|
expectedValue *bool
|
||||||
|
expectedWarning string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "preset enabled, bundle in Workspace, databricks runtime",
|
||||||
|
bundlePath: workspacePath,
|
||||||
|
ctx: dbr.MockRuntime(testContext, true),
|
||||||
|
mode: config.Production,
|
||||||
|
initialValue: &enabled,
|
||||||
|
expectedValue: &enabled,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "preset enabled, bundle not in Workspace, databricks runtime",
|
||||||
|
bundlePath: "/Users/user.name@company.com",
|
||||||
|
ctx: dbr.MockRuntime(testContext, true),
|
||||||
|
mode: config.Production,
|
||||||
|
initialValue: &enabled,
|
||||||
|
expectedValue: &disabled,
|
||||||
|
expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "preset enabled, bundle in Workspace, not databricks runtime",
|
||||||
|
bundlePath: workspacePath,
|
||||||
|
ctx: dbr.MockRuntime(testContext, false),
|
||||||
|
mode: config.Production,
|
||||||
|
initialValue: &enabled,
|
||||||
|
expectedValue: &disabled,
|
||||||
|
expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "preset disabled, bundle in Workspace, databricks runtime",
|
||||||
|
bundlePath: workspacePath,
|
||||||
|
ctx: dbr.MockRuntime(testContext, true),
|
||||||
|
mode: config.Production,
|
||||||
|
initialValue: &disabled,
|
||||||
|
expectedValue: &disabled,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "preset nil, bundle in Workspace, databricks runtime",
|
||||||
|
bundlePath: workspacePath,
|
||||||
|
ctx: dbr.MockRuntime(testContext, true),
|
||||||
|
mode: config.Production,
|
||||||
|
initialValue: nil,
|
||||||
|
expectedValue: nil,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "preset nil, dev mode true, bundle in Workspace, databricks runtime",
|
||||||
|
bundlePath: workspacePath,
|
||||||
|
ctx: dbr.MockRuntime(testContext, true),
|
||||||
|
mode: config.Development,
|
||||||
|
initialValue: nil,
|
||||||
|
expectedValue: &enabled,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
SyncRootPath: tt.bundlePath,
|
||||||
|
Config: config.Root{
|
||||||
|
Presets: config.Presets{
|
||||||
|
SourceLinkedDeployment: tt.initialValue,
|
||||||
|
},
|
||||||
|
Bundle: config.Bundle{
|
||||||
|
Mode: tt.mode,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
bundletest.SetLocation(b, "presets.source_linked_deployment", []dyn.Location{{File: "databricks.yml"}})
|
||||||
|
diags := bundle.Apply(tt.ctx, b, mutator.ApplySourceLinkedDeploymentPreset())
|
||||||
|
if diags.HasError() {
|
||||||
|
t.Fatalf("unexpected error: %v", diags)
|
||||||
|
}
|
||||||
|
|
||||||
|
if tt.expectedWarning != "" {
|
||||||
|
require.Equal(t, tt.expectedWarning, diags[0].Summary)
|
||||||
|
require.NotEmpty(t, diags[0].Locations)
|
||||||
|
}
|
||||||
|
|
||||||
|
require.Equal(t, tt.expectedValue, b.Config.Presets.SourceLinkedDeployment)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
|
@ -6,7 +6,6 @@ import (
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/libs/dbr"
|
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/databricks/cli/libs/iamutil"
|
"github.com/databricks/cli/libs/iamutil"
|
||||||
|
@ -58,14 +57,6 @@ func transformDevelopmentMode(ctx context.Context, b *bundle.Bundle) {
|
||||||
t.TriggerPauseStatus = config.Paused
|
t.TriggerPauseStatus = config.Paused
|
||||||
}
|
}
|
||||||
|
|
||||||
if !config.IsExplicitlyDisabled(t.SourceLinkedDeployment) {
|
|
||||||
isInWorkspace := strings.HasPrefix(b.SyncRootPath, "/Workspace/")
|
|
||||||
if isInWorkspace && dbr.RunsOnRuntime(ctx) {
|
|
||||||
enabled := true
|
|
||||||
t.SourceLinkedDeployment = &enabled
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !config.IsExplicitlyDisabled(t.PipelinesDevelopment) {
|
if !config.IsExplicitlyDisabled(t.PipelinesDevelopment) {
|
||||||
enabled := true
|
enabled := true
|
||||||
t.PipelinesDevelopment = &enabled
|
t.PipelinesDevelopment = &enabled
|
||||||
|
|
|
@ -3,14 +3,12 @@ package mutator
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"reflect"
|
"reflect"
|
||||||
"runtime"
|
|
||||||
"slices"
|
"slices"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/resources"
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
"github.com/databricks/cli/libs/dbr"
|
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
"github.com/databricks/cli/libs/tags"
|
"github.com/databricks/cli/libs/tags"
|
||||||
"github.com/databricks/cli/libs/vfs"
|
"github.com/databricks/cli/libs/vfs"
|
||||||
|
@ -540,32 +538,3 @@ func TestPipelinesDevelopmentDisabled(t *testing.T) {
|
||||||
|
|
||||||
assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
|
assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSourceLinkedDeploymentEnabled(t *testing.T) {
|
|
||||||
b, diags := processSourceLinkedBundle(t, true)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
assert.True(t, *b.Config.Presets.SourceLinkedDeployment)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSourceLinkedDeploymentDisabled(t *testing.T) {
|
|
||||||
b, diags := processSourceLinkedBundle(t, false)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
assert.False(t, *b.Config.Presets.SourceLinkedDeployment)
|
|
||||||
}
|
|
||||||
|
|
||||||
func processSourceLinkedBundle(t *testing.T, presetEnabled bool) (*bundle.Bundle, diag.Diagnostics) {
|
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
|
|
||||||
}
|
|
||||||
|
|
||||||
b := mockBundle(config.Development)
|
|
||||||
|
|
||||||
workspacePath := "/Workspace/lennart@company.com/"
|
|
||||||
b.SyncRootPath = workspacePath
|
|
||||||
b.Config.Presets.SourceLinkedDeployment = &presetEnabled
|
|
||||||
|
|
||||||
ctx := dbr.MockRuntime(context.Background(), true)
|
|
||||||
m := bundle.Seq(ProcessTargetMode(), ApplyPresets())
|
|
||||||
diags := bundle.Apply(ctx, b, m)
|
|
||||||
return b, diags
|
|
||||||
}
|
|
||||||
|
|
|
@ -51,6 +51,11 @@ func Initialize() bundle.Mutator {
|
||||||
mutator.RewriteWorkspacePrefix(),
|
mutator.RewriteWorkspacePrefix(),
|
||||||
|
|
||||||
mutator.SetVariables(),
|
mutator.SetVariables(),
|
||||||
|
|
||||||
|
// This mutator needs to be run before variable interpolation because it affects
|
||||||
|
// how workspace variables are resolved.
|
||||||
|
mutator.ApplySourceLinkedDeploymentPreset(),
|
||||||
|
|
||||||
// Intentionally placed before ResolveVariableReferencesInLookup, ResolveResourceReferences,
|
// Intentionally placed before ResolveVariableReferencesInLookup, ResolveResourceReferences,
|
||||||
// ResolveVariableReferencesInComplexVariables and ResolveVariableReferences.
|
// ResolveVariableReferencesInComplexVariables and ResolveVariableReferences.
|
||||||
// See what is expected in PythonMutatorPhaseInit doc
|
// See what is expected in PythonMutatorPhaseInit doc
|
||||||
|
|
Loading…
Reference in New Issue