mirror of https://github.com/databricks/cli.git
Handle `${workspace.file_path}` references in source-linked deployments (#2046)
## Changes 1. Updates `workspace.file_path` during source-linked deployment to address cases like this https://github.com/databricks/bundle-examples/blob/main/default_python/resources/default_python_pipeline.yml#L13 2. Updates `workspace.file_path` in `metadata.json` 3. Prints warning for users when `workspace.file_path` is explicitly set but deploy is running in source-linked mode ## Tests Unit test
This commit is contained in:
parent
185bbd28e4
commit
0289becea8
|
@ -9,7 +9,6 @@ import (
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/libs/dbr"
|
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/databricks/cli/libs/textutil"
|
"github.com/databricks/cli/libs/textutil"
|
||||||
|
@ -222,27 +221,6 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
|
||||||
dashboard.DisplayName = prefix + dashboard.DisplayName
|
dashboard.DisplayName = prefix + dashboard.DisplayName
|
||||||
}
|
}
|
||||||
|
|
||||||
if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) {
|
|
||||||
isDatabricksWorkspace := dbr.RunsOnRuntime(ctx) && strings.HasPrefix(b.SyncRootPath, "/Workspace/")
|
|
||||||
if !isDatabricksWorkspace {
|
|
||||||
target := b.Config.Bundle.Target
|
|
||||||
path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment"))
|
|
||||||
diags = diags.Append(
|
|
||||||
diag.Diagnostic{
|
|
||||||
Severity: diag.Warning,
|
|
||||||
Summary: "source-linked deployment is available only in the Databricks Workspace",
|
|
||||||
Paths: []dyn.Path{
|
|
||||||
path,
|
|
||||||
},
|
|
||||||
Locations: b.Config.GetLocations(path[2:].String()),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
disabled := false
|
|
||||||
b.Config.Presets.SourceLinkedDeployment = &disabled
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return diags
|
return diags
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -2,16 +2,12 @@ package mutator_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"runtime"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/mutator"
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
"github.com/databricks/cli/bundle/config/resources"
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
"github.com/databricks/cli/bundle/internal/bundletest"
|
|
||||||
"github.com/databricks/cli/libs/dbr"
|
|
||||||
"github.com/databricks/cli/libs/dyn"
|
|
||||||
"github.com/databricks/databricks-sdk-go/service/catalog"
|
"github.com/databricks/databricks-sdk-go/service/catalog"
|
||||||
"github.com/databricks/databricks-sdk-go/service/jobs"
|
"github.com/databricks/databricks-sdk-go/service/jobs"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
@ -398,87 +394,3 @@ func TestApplyPresetsResourceNotDefined(t *testing.T) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
|
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
|
|
||||||
}
|
|
||||||
|
|
||||||
testContext := context.Background()
|
|
||||||
enabled := true
|
|
||||||
disabled := false
|
|
||||||
workspacePath := "/Workspace/user.name@company.com"
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
bundlePath string
|
|
||||||
ctx context.Context
|
|
||||||
name string
|
|
||||||
initialValue *bool
|
|
||||||
expectedValue *bool
|
|
||||||
expectedWarning string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "preset enabled, bundle in Workspace, databricks runtime",
|
|
||||||
bundlePath: workspacePath,
|
|
||||||
ctx: dbr.MockRuntime(testContext, true),
|
|
||||||
initialValue: &enabled,
|
|
||||||
expectedValue: &enabled,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "preset enabled, bundle not in Workspace, databricks runtime",
|
|
||||||
bundlePath: "/Users/user.name@company.com",
|
|
||||||
ctx: dbr.MockRuntime(testContext, true),
|
|
||||||
initialValue: &enabled,
|
|
||||||
expectedValue: &disabled,
|
|
||||||
expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "preset enabled, bundle in Workspace, not databricks runtime",
|
|
||||||
bundlePath: workspacePath,
|
|
||||||
ctx: dbr.MockRuntime(testContext, false),
|
|
||||||
initialValue: &enabled,
|
|
||||||
expectedValue: &disabled,
|
|
||||||
expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "preset disabled, bundle in Workspace, databricks runtime",
|
|
||||||
bundlePath: workspacePath,
|
|
||||||
ctx: dbr.MockRuntime(testContext, true),
|
|
||||||
initialValue: &disabled,
|
|
||||||
expectedValue: &disabled,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "preset nil, bundle in Workspace, databricks runtime",
|
|
||||||
bundlePath: workspacePath,
|
|
||||||
ctx: dbr.MockRuntime(testContext, true),
|
|
||||||
initialValue: nil,
|
|
||||||
expectedValue: nil,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
SyncRootPath: tt.bundlePath,
|
|
||||||
Config: config.Root{
|
|
||||||
Presets: config.Presets{
|
|
||||||
SourceLinkedDeployment: tt.initialValue,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
bundletest.SetLocation(b, "presets.source_linked_deployment", []dyn.Location{{File: "databricks.yml"}})
|
|
||||||
diags := bundle.Apply(tt.ctx, b, mutator.ApplyPresets())
|
|
||||||
if diags.HasError() {
|
|
||||||
t.Fatalf("unexpected error: %v", diags)
|
|
||||||
}
|
|
||||||
|
|
||||||
if tt.expectedWarning != "" {
|
|
||||||
require.Equal(t, tt.expectedWarning, diags[0].Summary)
|
|
||||||
require.NotEmpty(t, diags[0].Locations)
|
|
||||||
}
|
|
||||||
|
|
||||||
require.Equal(t, tt.expectedValue, b.Config.Presets.SourceLinkedDeployment)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
@ -0,0 +1,75 @@
|
||||||
|
package mutator
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config"
|
||||||
|
"github.com/databricks/cli/libs/dbr"
|
||||||
|
"github.com/databricks/cli/libs/diag"
|
||||||
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
)
|
||||||
|
|
||||||
|
type applySourceLinkedDeploymentPreset struct{}
|
||||||
|
|
||||||
|
// Apply source-linked deployment preset
|
||||||
|
func ApplySourceLinkedDeploymentPreset() *applySourceLinkedDeploymentPreset {
|
||||||
|
return &applySourceLinkedDeploymentPreset{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *applySourceLinkedDeploymentPreset) Name() string {
|
||||||
|
return "ApplySourceLinkedDeploymentPreset"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||||
|
if config.IsExplicitlyDisabled(b.Config.Presets.SourceLinkedDeployment) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var diags diag.Diagnostics
|
||||||
|
isDatabricksWorkspace := dbr.RunsOnRuntime(ctx) && strings.HasPrefix(b.SyncRootPath, "/Workspace/")
|
||||||
|
target := b.Config.Bundle.Target
|
||||||
|
|
||||||
|
if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) {
|
||||||
|
if !isDatabricksWorkspace {
|
||||||
|
path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment"))
|
||||||
|
diags = diags.Append(
|
||||||
|
diag.Diagnostic{
|
||||||
|
Severity: diag.Warning,
|
||||||
|
Summary: "source-linked deployment is available only in the Databricks Workspace",
|
||||||
|
Paths: []dyn.Path{
|
||||||
|
path,
|
||||||
|
},
|
||||||
|
Locations: b.Config.GetLocations(path[2:].String()),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
disabled := false
|
||||||
|
b.Config.Presets.SourceLinkedDeployment = &disabled
|
||||||
|
return diags
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if isDatabricksWorkspace && b.Config.Bundle.Mode == config.Development {
|
||||||
|
enabled := true
|
||||||
|
b.Config.Presets.SourceLinkedDeployment = &enabled
|
||||||
|
}
|
||||||
|
|
||||||
|
if b.Config.Workspace.FilePath != "" && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
|
||||||
|
path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("workspace"), dyn.Key("file_path"))
|
||||||
|
|
||||||
|
diags = diags.Append(
|
||||||
|
diag.Diagnostic{
|
||||||
|
Severity: diag.Warning,
|
||||||
|
Summary: "workspace.file_path setting will be ignored in source-linked deployment mode",
|
||||||
|
Paths: []dyn.Path{
|
||||||
|
path[2:],
|
||||||
|
},
|
||||||
|
Locations: b.Config.GetLocations(path[2:].String()),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return diags
|
||||||
|
}
|
|
@ -0,0 +1,122 @@
|
||||||
|
package mutator_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"runtime"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config"
|
||||||
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
|
"github.com/databricks/cli/libs/dbr"
|
||||||
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
|
||||||
|
}
|
||||||
|
|
||||||
|
testContext := context.Background()
|
||||||
|
enabled := true
|
||||||
|
disabled := false
|
||||||
|
workspacePath := "/Workspace/user.name@company.com"
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
ctx context.Context
|
||||||
|
mutateBundle func(b *bundle.Bundle)
|
||||||
|
initialValue *bool
|
||||||
|
expectedValue *bool
|
||||||
|
expectedWarning string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "preset enabled, bundle in Workspace, databricks runtime",
|
||||||
|
ctx: dbr.MockRuntime(testContext, true),
|
||||||
|
initialValue: &enabled,
|
||||||
|
expectedValue: &enabled,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "preset enabled, bundle not in Workspace, databricks runtime",
|
||||||
|
ctx: dbr.MockRuntime(testContext, true),
|
||||||
|
mutateBundle: func(b *bundle.Bundle) {
|
||||||
|
b.SyncRootPath = "/Users/user.name@company.com"
|
||||||
|
},
|
||||||
|
initialValue: &enabled,
|
||||||
|
expectedValue: &disabled,
|
||||||
|
expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "preset enabled, bundle in Workspace, not databricks runtime",
|
||||||
|
ctx: dbr.MockRuntime(testContext, false),
|
||||||
|
initialValue: &enabled,
|
||||||
|
expectedValue: &disabled,
|
||||||
|
expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "preset disabled, bundle in Workspace, databricks runtime",
|
||||||
|
ctx: dbr.MockRuntime(testContext, true),
|
||||||
|
initialValue: &disabled,
|
||||||
|
expectedValue: &disabled,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "preset nil, bundle in Workspace, databricks runtime",
|
||||||
|
ctx: dbr.MockRuntime(testContext, true),
|
||||||
|
initialValue: nil,
|
||||||
|
expectedValue: nil,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "preset nil, dev mode true, bundle in Workspace, databricks runtime",
|
||||||
|
ctx: dbr.MockRuntime(testContext, true),
|
||||||
|
mutateBundle: func(b *bundle.Bundle) {
|
||||||
|
b.Config.Bundle.Mode = config.Development
|
||||||
|
},
|
||||||
|
initialValue: nil,
|
||||||
|
expectedValue: &enabled,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "preset enabled, workspace.file_path is defined by user",
|
||||||
|
ctx: dbr.MockRuntime(testContext, true),
|
||||||
|
mutateBundle: func(b *bundle.Bundle) {
|
||||||
|
b.Config.Workspace.FilePath = "file_path"
|
||||||
|
},
|
||||||
|
initialValue: &enabled,
|
||||||
|
expectedValue: &enabled,
|
||||||
|
expectedWarning: "workspace.file_path setting will be ignored in source-linked deployment mode",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
SyncRootPath: workspacePath,
|
||||||
|
Config: config.Root{
|
||||||
|
Presets: config.Presets{
|
||||||
|
SourceLinkedDeployment: tt.initialValue,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if tt.mutateBundle != nil {
|
||||||
|
tt.mutateBundle(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
bundletest.SetLocation(b, "presets.source_linked_deployment", []dyn.Location{{File: "databricks.yml"}})
|
||||||
|
bundletest.SetLocation(b, "workspace.file_path", []dyn.Location{{File: "databricks.yml"}})
|
||||||
|
|
||||||
|
diags := bundle.Apply(tt.ctx, b, mutator.ApplySourceLinkedDeploymentPreset())
|
||||||
|
if diags.HasError() {
|
||||||
|
t.Fatalf("unexpected error: %v", diags)
|
||||||
|
}
|
||||||
|
|
||||||
|
if tt.expectedWarning != "" {
|
||||||
|
require.Equal(t, tt.expectedWarning, diags[0].Summary)
|
||||||
|
require.NotEmpty(t, diags[0].Locations)
|
||||||
|
}
|
||||||
|
|
||||||
|
require.Equal(t, tt.expectedValue, b.Config.Presets.SourceLinkedDeployment)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
|
@ -6,7 +6,6 @@ import (
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/libs/dbr"
|
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/databricks/cli/libs/iamutil"
|
"github.com/databricks/cli/libs/iamutil"
|
||||||
|
@ -58,14 +57,6 @@ func transformDevelopmentMode(ctx context.Context, b *bundle.Bundle) {
|
||||||
t.TriggerPauseStatus = config.Paused
|
t.TriggerPauseStatus = config.Paused
|
||||||
}
|
}
|
||||||
|
|
||||||
if !config.IsExplicitlyDisabled(t.SourceLinkedDeployment) {
|
|
||||||
isInWorkspace := strings.HasPrefix(b.SyncRootPath, "/Workspace/")
|
|
||||||
if isInWorkspace && dbr.RunsOnRuntime(ctx) {
|
|
||||||
enabled := true
|
|
||||||
t.SourceLinkedDeployment = &enabled
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !config.IsExplicitlyDisabled(t.PipelinesDevelopment) {
|
if !config.IsExplicitlyDisabled(t.PipelinesDevelopment) {
|
||||||
enabled := true
|
enabled := true
|
||||||
t.PipelinesDevelopment = &enabled
|
t.PipelinesDevelopment = &enabled
|
||||||
|
|
|
@ -3,14 +3,12 @@ package mutator
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"reflect"
|
"reflect"
|
||||||
"runtime"
|
|
||||||
"slices"
|
"slices"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/resources"
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
"github.com/databricks/cli/libs/dbr"
|
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
"github.com/databricks/cli/libs/tags"
|
"github.com/databricks/cli/libs/tags"
|
||||||
"github.com/databricks/cli/libs/vfs"
|
"github.com/databricks/cli/libs/vfs"
|
||||||
|
@ -540,32 +538,3 @@ func TestPipelinesDevelopmentDisabled(t *testing.T) {
|
||||||
|
|
||||||
assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
|
assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSourceLinkedDeploymentEnabled(t *testing.T) {
|
|
||||||
b, diags := processSourceLinkedBundle(t, true)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
assert.True(t, *b.Config.Presets.SourceLinkedDeployment)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSourceLinkedDeploymentDisabled(t *testing.T) {
|
|
||||||
b, diags := processSourceLinkedBundle(t, false)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
assert.False(t, *b.Config.Presets.SourceLinkedDeployment)
|
|
||||||
}
|
|
||||||
|
|
||||||
func processSourceLinkedBundle(t *testing.T, presetEnabled bool) (*bundle.Bundle, diag.Diagnostics) {
|
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
|
|
||||||
}
|
|
||||||
|
|
||||||
b := mockBundle(config.Development)
|
|
||||||
|
|
||||||
workspacePath := "/Workspace/lennart@company.com/"
|
|
||||||
b.SyncRootPath = workspacePath
|
|
||||||
b.Config.Presets.SourceLinkedDeployment = &presetEnabled
|
|
||||||
|
|
||||||
ctx := dbr.MockRuntime(context.Background(), true)
|
|
||||||
m := bundle.Seq(ProcessTargetMode(), ApplyPresets())
|
|
||||||
diags := bundle.Apply(ctx, b, m)
|
|
||||||
return b, diags
|
|
||||||
}
|
|
||||||
|
|
|
@ -5,6 +5,7 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/variable"
|
"github.com/databricks/cli/bundle/config/variable"
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
@ -15,7 +16,7 @@ import (
|
||||||
type resolveVariableReferences struct {
|
type resolveVariableReferences struct {
|
||||||
prefixes []string
|
prefixes []string
|
||||||
pattern dyn.Pattern
|
pattern dyn.Pattern
|
||||||
lookupFn func(dyn.Value, dyn.Path) (dyn.Value, error)
|
lookupFn func(dyn.Value, dyn.Path, *bundle.Bundle) (dyn.Value, error)
|
||||||
skipFn func(dyn.Value) bool
|
skipFn func(dyn.Value) bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -44,16 +45,21 @@ func ResolveVariableReferencesInComplexVariables() bundle.Mutator {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func lookup(v dyn.Value, path dyn.Path) (dyn.Value, error) {
|
func lookup(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) {
|
||||||
|
if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
|
||||||
|
if path.String() == "workspace.file_path" {
|
||||||
|
return dyn.V(b.SyncRootPath), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
// Future opportunity: if we lookup this path in both the given root
|
// Future opportunity: if we lookup this path in both the given root
|
||||||
// and the synthesized root, we know if it was explicitly set or implied to be empty.
|
// and the synthesized root, we know if it was explicitly set or implied to be empty.
|
||||||
// Then we can emit a warning if it was not explicitly set.
|
// Then we can emit a warning if it was not explicitly set.
|
||||||
return dyn.GetByPath(v, path)
|
return dyn.GetByPath(v, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
func lookupForComplexVariables(v dyn.Value, path dyn.Path) (dyn.Value, error) {
|
func lookupForComplexVariables(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) {
|
||||||
if path[0].Key() != "variables" {
|
if path[0].Key() != "variables" {
|
||||||
return lookup(v, path)
|
return lookup(v, path, b)
|
||||||
}
|
}
|
||||||
|
|
||||||
varV, err := dyn.GetByPath(v, path[:len(path)-1])
|
varV, err := dyn.GetByPath(v, path[:len(path)-1])
|
||||||
|
@ -71,7 +77,7 @@ func lookupForComplexVariables(v dyn.Value, path dyn.Path) (dyn.Value, error) {
|
||||||
return dyn.InvalidValue, errors.New("complex variables cannot contain references to another complex variables")
|
return dyn.InvalidValue, errors.New("complex variables cannot contain references to another complex variables")
|
||||||
}
|
}
|
||||||
|
|
||||||
return lookup(v, path)
|
return lookup(v, path, b)
|
||||||
}
|
}
|
||||||
|
|
||||||
func skipResolvingInNonComplexVariables(v dyn.Value) bool {
|
func skipResolvingInNonComplexVariables(v dyn.Value) bool {
|
||||||
|
@ -83,9 +89,9 @@ func skipResolvingInNonComplexVariables(v dyn.Value) bool {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func lookupForVariables(v dyn.Value, path dyn.Path) (dyn.Value, error) {
|
func lookupForVariables(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) {
|
||||||
if path[0].Key() != "variables" {
|
if path[0].Key() != "variables" {
|
||||||
return lookup(v, path)
|
return lookup(v, path, b)
|
||||||
}
|
}
|
||||||
|
|
||||||
varV, err := dyn.GetByPath(v, path[:len(path)-1])
|
varV, err := dyn.GetByPath(v, path[:len(path)-1])
|
||||||
|
@ -103,7 +109,7 @@ func lookupForVariables(v dyn.Value, path dyn.Path) (dyn.Value, error) {
|
||||||
return dyn.InvalidValue, errors.New("lookup variables cannot contain references to another lookup variables")
|
return dyn.InvalidValue, errors.New("lookup variables cannot contain references to another lookup variables")
|
||||||
}
|
}
|
||||||
|
|
||||||
return lookup(v, path)
|
return lookup(v, path, b)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (*resolveVariableReferences) Name() string {
|
func (*resolveVariableReferences) Name() string {
|
||||||
|
@ -125,6 +131,7 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle)
|
||||||
varPath := dyn.NewPath(dyn.Key("var"))
|
varPath := dyn.NewPath(dyn.Key("var"))
|
||||||
|
|
||||||
var diags diag.Diagnostics
|
var diags diag.Diagnostics
|
||||||
|
|
||||||
err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
|
err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
|
||||||
// Synthesize a copy of the root that has all fields that are present in the type
|
// Synthesize a copy of the root that has all fields that are present in the type
|
||||||
// but not set in the dynamic value set to their corresponding empty value.
|
// but not set in the dynamic value set to their corresponding empty value.
|
||||||
|
@ -167,7 +174,7 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle)
|
||||||
if m.skipFn != nil && m.skipFn(v) {
|
if m.skipFn != nil && m.skipFn(v) {
|
||||||
return dyn.InvalidValue, dynvar.ErrSkipResolution
|
return dyn.InvalidValue, dynvar.ErrSkipResolution
|
||||||
}
|
}
|
||||||
return m.lookupFn(normalized, path)
|
return m.lookupFn(normalized, path, b)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -12,6 +12,7 @@ import (
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
"github.com/databricks/databricks-sdk-go/service/compute"
|
||||||
"github.com/databricks/databricks-sdk-go/service/jobs"
|
"github.com/databricks/databricks-sdk-go/service/jobs"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/pipelines"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
@ -434,3 +435,57 @@ func TestResolveComplexVariableWithVarReference(t *testing.T) {
|
||||||
require.NoError(t, diags.Error())
|
require.NoError(t, diags.Error())
|
||||||
require.Equal(t, "cicd_template==1.0.0", b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].Libraries[0].Pypi.Package)
|
require.Equal(t, "cicd_template==1.0.0", b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].Libraries[0].Pypi.Package)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) {
|
||||||
|
testCases := []struct {
|
||||||
|
enabled bool
|
||||||
|
assert func(t *testing.T, b *bundle.Bundle)
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
true,
|
||||||
|
func(t *testing.T, b *bundle.Bundle) {
|
||||||
|
// Variables that use workspace file path should have SyncRootValue during resolution phase
|
||||||
|
require.Equal(t, "sync/root/path", b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Configuration["source"])
|
||||||
|
|
||||||
|
// The file path itself should remain the same
|
||||||
|
require.Equal(t, "file/path", b.Config.Workspace.FilePath)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
false,
|
||||||
|
func(t *testing.T, b *bundle.Bundle) {
|
||||||
|
require.Equal(t, "file/path", b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Configuration["source"])
|
||||||
|
require.Equal(t, "file/path", b.Config.Workspace.FilePath)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, testCase := range testCases {
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
SyncRootPath: "sync/root/path",
|
||||||
|
Config: config.Root{
|
||||||
|
Presets: config.Presets{
|
||||||
|
SourceLinkedDeployment: &testCase.enabled,
|
||||||
|
},
|
||||||
|
Workspace: config.Workspace{
|
||||||
|
FilePath: "file/path",
|
||||||
|
},
|
||||||
|
Resources: config.Resources{
|
||||||
|
Pipelines: map[string]*resources.Pipeline{
|
||||||
|
"pipeline1": {
|
||||||
|
PipelineSpec: &pipelines.PipelineSpec{
|
||||||
|
Configuration: map[string]string{
|
||||||
|
"source": "${workspace.file_path}",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
diags := bundle.Apply(context.Background(), b, ResolveVariableReferences("workspace"))
|
||||||
|
require.NoError(t, diags.Error())
|
||||||
|
testCase.assert(t, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -54,5 +54,8 @@ func (m *compute) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||||
|
|
||||||
// Set file upload destination of the bundle in metadata
|
// Set file upload destination of the bundle in metadata
|
||||||
b.Metadata.Config.Workspace.FilePath = b.Config.Workspace.FilePath
|
b.Metadata.Config.Workspace.FilePath = b.Config.Workspace.FilePath
|
||||||
|
if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
|
||||||
|
b.Metadata.Config.Workspace.FilePath = b.SyncRootPath
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -97,3 +97,24 @@ func TestComputeMetadataMutator(t *testing.T) {
|
||||||
|
|
||||||
assert.Equal(t, expectedMetadata, b.Metadata)
|
assert.Equal(t, expectedMetadata, b.Metadata)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestComputeMetadataMutatorSourceLinked(t *testing.T) {
|
||||||
|
syncRootPath := "/Users/shreyas.goenka@databricks.com/source"
|
||||||
|
enabled := true
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
SyncRootPath: syncRootPath,
|
||||||
|
Config: config.Root{
|
||||||
|
Presets: config.Presets{
|
||||||
|
SourceLinkedDeployment: &enabled,
|
||||||
|
},
|
||||||
|
Workspace: config.Workspace{
|
||||||
|
FilePath: "/Users/shreyas.goenka@databricks.com/files",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
diags := bundle.Apply(context.Background(), b, Compute())
|
||||||
|
require.NoError(t, diags.Error())
|
||||||
|
|
||||||
|
assert.Equal(t, syncRootPath, b.Metadata.Config.Workspace.FilePath)
|
||||||
|
}
|
||||||
|
|
|
@ -41,6 +41,10 @@ func Initialize() bundle.Mutator {
|
||||||
mutator.PopulateCurrentUser(),
|
mutator.PopulateCurrentUser(),
|
||||||
mutator.LoadGitDetails(),
|
mutator.LoadGitDetails(),
|
||||||
|
|
||||||
|
// This mutator needs to be run before variable interpolation and defining default workspace paths
|
||||||
|
// because it affects how workspace variables are resolved.
|
||||||
|
mutator.ApplySourceLinkedDeploymentPreset(),
|
||||||
|
|
||||||
mutator.DefineDefaultWorkspaceRoot(),
|
mutator.DefineDefaultWorkspaceRoot(),
|
||||||
mutator.ExpandWorkspaceRoot(),
|
mutator.ExpandWorkspaceRoot(),
|
||||||
mutator.DefineDefaultWorkspacePaths(),
|
mutator.DefineDefaultWorkspacePaths(),
|
||||||
|
@ -51,6 +55,7 @@ func Initialize() bundle.Mutator {
|
||||||
mutator.RewriteWorkspacePrefix(),
|
mutator.RewriteWorkspacePrefix(),
|
||||||
|
|
||||||
mutator.SetVariables(),
|
mutator.SetVariables(),
|
||||||
|
|
||||||
// Intentionally placed before ResolveVariableReferencesInLookup, ResolveResourceReferences,
|
// Intentionally placed before ResolveVariableReferencesInLookup, ResolveResourceReferences,
|
||||||
// ResolveVariableReferencesInComplexVariables and ResolveVariableReferences.
|
// ResolveVariableReferencesInComplexVariables and ResolveVariableReferences.
|
||||||
// See what is expected in PythonMutatorPhaseInit doc
|
// See what is expected in PythonMutatorPhaseInit doc
|
||||||
|
|
|
@ -7,6 +7,7 @@ import (
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/config/mutator"
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
"github.com/databricks/cli/bundle/phases"
|
"github.com/databricks/cli/bundle/phases"
|
||||||
|
"github.com/databricks/cli/libs/dbr"
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
"github.com/databricks/databricks-sdk-go/config"
|
"github.com/databricks/databricks-sdk-go/config"
|
||||||
"github.com/databricks/databricks-sdk-go/experimental/mocks"
|
"github.com/databricks/databricks-sdk-go/experimental/mocks"
|
||||||
|
@ -66,7 +67,7 @@ func initializeTarget(t *testing.T, path, env string) (*bundle.Bundle, diag.Diag
|
||||||
b := load(t, path)
|
b := load(t, path)
|
||||||
configureMock(t, b)
|
configureMock(t, b)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := dbr.MockRuntime(context.Background(), false)
|
||||||
diags := bundle.Apply(ctx, b, bundle.Seq(
|
diags := bundle.Apply(ctx, b, bundle.Seq(
|
||||||
mutator.SelectTarget(env),
|
mutator.SelectTarget(env),
|
||||||
phases.Initialize(),
|
phases.Initialize(),
|
||||||
|
|
|
@ -16,6 +16,7 @@ import (
|
||||||
"github.com/databricks/cli/bundle/phases"
|
"github.com/databricks/cli/bundle/phases"
|
||||||
"github.com/databricks/cli/cmd/root"
|
"github.com/databricks/cli/cmd/root"
|
||||||
"github.com/databricks/cli/internal/testutil"
|
"github.com/databricks/cli/internal/testutil"
|
||||||
|
"github.com/databricks/cli/libs/dbr"
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
"github.com/databricks/cli/libs/filer"
|
"github.com/databricks/cli/libs/filer"
|
||||||
"github.com/databricks/cli/libs/tags"
|
"github.com/databricks/cli/libs/tags"
|
||||||
|
@ -39,7 +40,7 @@ func assertFilePermissions(t *testing.T, path string, perm fs.FileMode) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func assertBuiltinTemplateValid(t *testing.T, template string, settings map[string]any, target string, isServicePrincipal, build bool, tempDir string) {
|
func assertBuiltinTemplateValid(t *testing.T, template string, settings map[string]any, target string, isServicePrincipal, build bool, tempDir string) {
|
||||||
ctx := context.Background()
|
ctx := dbr.MockRuntime(context.Background(), false)
|
||||||
|
|
||||||
templateFS, err := fs.Sub(builtinTemplates, path.Join("templates", template))
|
templateFS, err := fs.Sub(builtinTemplates, path.Join("templates", template))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
Loading…
Reference in New Issue