From 0289becea854dff5a89ee1c14809a3cc3cba998b Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Wed, 8 Jan 2025 13:43:56 +0100 Subject: [PATCH] Handle `${workspace.file_path}` references in source-linked deployments (#2046) ## Changes 1. Updates `workspace.file_path` during source-linked deployment to address cases like this https://github.com/databricks/bundle-examples/blob/main/default_python/resources/default_python_pipeline.yml#L13 2. Updates `workspace.file_path` in `metadata.json` 3. Prints warning for users when `workspace.file_path` is explicitly set but deploy is running in source-linked mode ## Tests Unit test --- bundle/config/mutator/apply_presets.go | 22 ---- bundle/config/mutator/apply_presets_test.go | 88 ------------- .../apply_source_linked_deployment_preset.go | 75 +++++++++++ ...ly_source_linked_deployment_preset_test.go | 122 ++++++++++++++++++ bundle/config/mutator/process_target_mode.go | 9 -- .../mutator/process_target_mode_test.go | 31 ----- .../mutator/resolve_variable_references.go | 25 ++-- .../resolve_variable_references_test.go | 55 ++++++++ bundle/deploy/metadata/compute.go | 3 + bundle/deploy/metadata/compute_test.go | 21 +++ bundle/phases/initialize.go | 5 + bundle/tests/loader.go | 3 +- libs/template/renderer_test.go | 3 +- 13 files changed, 301 insertions(+), 161 deletions(-) create mode 100644 bundle/config/mutator/apply_source_linked_deployment_preset.go create mode 100644 bundle/config/mutator/apply_source_linked_deployment_preset_test.go diff --git a/bundle/config/mutator/apply_presets.go b/bundle/config/mutator/apply_presets.go index 381703756..59b8547be 100644 --- a/bundle/config/mutator/apply_presets.go +++ b/bundle/config/mutator/apply_presets.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/dbr" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/textutil" @@ -222,27 +221,6 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos dashboard.DisplayName = prefix + dashboard.DisplayName } - if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) { - isDatabricksWorkspace := dbr.RunsOnRuntime(ctx) && strings.HasPrefix(b.SyncRootPath, "/Workspace/") - if !isDatabricksWorkspace { - target := b.Config.Bundle.Target - path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment")) - diags = diags.Append( - diag.Diagnostic{ - Severity: diag.Warning, - Summary: "source-linked deployment is available only in the Databricks Workspace", - Paths: []dyn.Path{ - path, - }, - Locations: b.Config.GetLocations(path[2:].String()), - }, - ) - - disabled := false - b.Config.Presets.SourceLinkedDeployment = &disabled - } - } - return diags } diff --git a/bundle/config/mutator/apply_presets_test.go b/bundle/config/mutator/apply_presets_test.go index c26f20383..5e3f942cc 100644 --- a/bundle/config/mutator/apply_presets_test.go +++ b/bundle/config/mutator/apply_presets_test.go @@ -2,16 +2,12 @@ package mutator_test import ( "context" - "runtime" "testing" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/dbr" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" @@ -398,87 +394,3 @@ func TestApplyPresetsResourceNotDefined(t *testing.T) { }) } } - -func TestApplyPresetsSourceLinkedDeployment(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace") - } - - testContext := context.Background() - enabled := true - disabled := false - workspacePath := "/Workspace/user.name@company.com" - - tests := []struct { - bundlePath string - ctx context.Context - name string - initialValue *bool - expectedValue *bool - expectedWarning string - }{ - { - name: "preset enabled, bundle in Workspace, databricks runtime", - bundlePath: workspacePath, - ctx: dbr.MockRuntime(testContext, true), - initialValue: &enabled, - expectedValue: &enabled, - }, - { - name: "preset enabled, bundle not in Workspace, databricks runtime", - bundlePath: "/Users/user.name@company.com", - ctx: dbr.MockRuntime(testContext, true), - initialValue: &enabled, - expectedValue: &disabled, - expectedWarning: "source-linked deployment is available only in the Databricks Workspace", - }, - { - name: "preset enabled, bundle in Workspace, not databricks runtime", - bundlePath: workspacePath, - ctx: dbr.MockRuntime(testContext, false), - initialValue: &enabled, - expectedValue: &disabled, - expectedWarning: "source-linked deployment is available only in the Databricks Workspace", - }, - { - name: "preset disabled, bundle in Workspace, databricks runtime", - bundlePath: workspacePath, - ctx: dbr.MockRuntime(testContext, true), - initialValue: &disabled, - expectedValue: &disabled, - }, - { - name: "preset nil, bundle in Workspace, databricks runtime", - bundlePath: workspacePath, - ctx: dbr.MockRuntime(testContext, true), - initialValue: nil, - expectedValue: nil, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b := &bundle.Bundle{ - SyncRootPath: tt.bundlePath, - Config: config.Root{ - Presets: config.Presets{ - SourceLinkedDeployment: tt.initialValue, - }, - }, - } - - bundletest.SetLocation(b, "presets.source_linked_deployment", []dyn.Location{{File: "databricks.yml"}}) - diags := bundle.Apply(tt.ctx, b, mutator.ApplyPresets()) - if diags.HasError() { - t.Fatalf("unexpected error: %v", diags) - } - - if tt.expectedWarning != "" { - require.Equal(t, tt.expectedWarning, diags[0].Summary) - require.NotEmpty(t, diags[0].Locations) - } - - require.Equal(t, tt.expectedValue, b.Config.Presets.SourceLinkedDeployment) - }) - } -} diff --git a/bundle/config/mutator/apply_source_linked_deployment_preset.go b/bundle/config/mutator/apply_source_linked_deployment_preset.go new file mode 100644 index 000000000..78ccc5322 --- /dev/null +++ b/bundle/config/mutator/apply_source_linked_deployment_preset.go @@ -0,0 +1,75 @@ +package mutator + +import ( + "context" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/dbr" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type applySourceLinkedDeploymentPreset struct{} + +// Apply source-linked deployment preset +func ApplySourceLinkedDeploymentPreset() *applySourceLinkedDeploymentPreset { + return &applySourceLinkedDeploymentPreset{} +} + +func (m *applySourceLinkedDeploymentPreset) Name() string { + return "ApplySourceLinkedDeploymentPreset" +} + +func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if config.IsExplicitlyDisabled(b.Config.Presets.SourceLinkedDeployment) { + return nil + } + + var diags diag.Diagnostics + isDatabricksWorkspace := dbr.RunsOnRuntime(ctx) && strings.HasPrefix(b.SyncRootPath, "/Workspace/") + target := b.Config.Bundle.Target + + if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) { + if !isDatabricksWorkspace { + path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment")) + diags = diags.Append( + diag.Diagnostic{ + Severity: diag.Warning, + Summary: "source-linked deployment is available only in the Databricks Workspace", + Paths: []dyn.Path{ + path, + }, + Locations: b.Config.GetLocations(path[2:].String()), + }, + ) + + disabled := false + b.Config.Presets.SourceLinkedDeployment = &disabled + return diags + } + } + + if isDatabricksWorkspace && b.Config.Bundle.Mode == config.Development { + enabled := true + b.Config.Presets.SourceLinkedDeployment = &enabled + } + + if b.Config.Workspace.FilePath != "" && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { + path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("workspace"), dyn.Key("file_path")) + + diags = diags.Append( + diag.Diagnostic{ + Severity: diag.Warning, + Summary: "workspace.file_path setting will be ignored in source-linked deployment mode", + Paths: []dyn.Path{ + path[2:], + }, + Locations: b.Config.GetLocations(path[2:].String()), + }, + ) + } + + return diags +} diff --git a/bundle/config/mutator/apply_source_linked_deployment_preset_test.go b/bundle/config/mutator/apply_source_linked_deployment_preset_test.go new file mode 100644 index 000000000..1b74fd8e9 --- /dev/null +++ b/bundle/config/mutator/apply_source_linked_deployment_preset_test.go @@ -0,0 +1,122 @@ +package mutator_test + +import ( + "context" + "runtime" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/dbr" + "github.com/databricks/cli/libs/dyn" + "github.com/stretchr/testify/require" +) + +func TestApplyPresetsSourceLinkedDeployment(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace") + } + + testContext := context.Background() + enabled := true + disabled := false + workspacePath := "/Workspace/user.name@company.com" + + tests := []struct { + name string + ctx context.Context + mutateBundle func(b *bundle.Bundle) + initialValue *bool + expectedValue *bool + expectedWarning string + }{ + { + name: "preset enabled, bundle in Workspace, databricks runtime", + ctx: dbr.MockRuntime(testContext, true), + initialValue: &enabled, + expectedValue: &enabled, + }, + { + name: "preset enabled, bundle not in Workspace, databricks runtime", + ctx: dbr.MockRuntime(testContext, true), + mutateBundle: func(b *bundle.Bundle) { + b.SyncRootPath = "/Users/user.name@company.com" + }, + initialValue: &enabled, + expectedValue: &disabled, + expectedWarning: "source-linked deployment is available only in the Databricks Workspace", + }, + { + name: "preset enabled, bundle in Workspace, not databricks runtime", + ctx: dbr.MockRuntime(testContext, false), + initialValue: &enabled, + expectedValue: &disabled, + expectedWarning: "source-linked deployment is available only in the Databricks Workspace", + }, + { + name: "preset disabled, bundle in Workspace, databricks runtime", + ctx: dbr.MockRuntime(testContext, true), + initialValue: &disabled, + expectedValue: &disabled, + }, + { + name: "preset nil, bundle in Workspace, databricks runtime", + ctx: dbr.MockRuntime(testContext, true), + initialValue: nil, + expectedValue: nil, + }, + { + name: "preset nil, dev mode true, bundle in Workspace, databricks runtime", + ctx: dbr.MockRuntime(testContext, true), + mutateBundle: func(b *bundle.Bundle) { + b.Config.Bundle.Mode = config.Development + }, + initialValue: nil, + expectedValue: &enabled, + }, + { + name: "preset enabled, workspace.file_path is defined by user", + ctx: dbr.MockRuntime(testContext, true), + mutateBundle: func(b *bundle.Bundle) { + b.Config.Workspace.FilePath = "file_path" + }, + initialValue: &enabled, + expectedValue: &enabled, + expectedWarning: "workspace.file_path setting will be ignored in source-linked deployment mode", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &bundle.Bundle{ + SyncRootPath: workspacePath, + Config: config.Root{ + Presets: config.Presets{ + SourceLinkedDeployment: tt.initialValue, + }, + }, + } + + if tt.mutateBundle != nil { + tt.mutateBundle(b) + } + + bundletest.SetLocation(b, "presets.source_linked_deployment", []dyn.Location{{File: "databricks.yml"}}) + bundletest.SetLocation(b, "workspace.file_path", []dyn.Location{{File: "databricks.yml"}}) + + diags := bundle.Apply(tt.ctx, b, mutator.ApplySourceLinkedDeploymentPreset()) + if diags.HasError() { + t.Fatalf("unexpected error: %v", diags) + } + + if tt.expectedWarning != "" { + require.Equal(t, tt.expectedWarning, diags[0].Summary) + require.NotEmpty(t, diags[0].Locations) + } + + require.Equal(t, tt.expectedValue, b.Config.Presets.SourceLinkedDeployment) + }) + } +} diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go index df0136fad..44b53681d 100644 --- a/bundle/config/mutator/process_target_mode.go +++ b/bundle/config/mutator/process_target_mode.go @@ -6,7 +6,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/dbr" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/iamutil" @@ -58,14 +57,6 @@ func transformDevelopmentMode(ctx context.Context, b *bundle.Bundle) { t.TriggerPauseStatus = config.Paused } - if !config.IsExplicitlyDisabled(t.SourceLinkedDeployment) { - isInWorkspace := strings.HasPrefix(b.SyncRootPath, "/Workspace/") - if isInWorkspace && dbr.RunsOnRuntime(ctx) { - enabled := true - t.SourceLinkedDeployment = &enabled - } - } - if !config.IsExplicitlyDisabled(t.PipelinesDevelopment) { enabled := true t.PipelinesDevelopment = &enabled diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index c299a7636..097c092a6 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -3,14 +3,12 @@ package mutator import ( "context" "reflect" - "runtime" "slices" "testing" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dbr" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/tags" "github.com/databricks/cli/libs/vfs" @@ -540,32 +538,3 @@ func TestPipelinesDevelopmentDisabled(t *testing.T) { assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) } - -func TestSourceLinkedDeploymentEnabled(t *testing.T) { - b, diags := processSourceLinkedBundle(t, true) - require.NoError(t, diags.Error()) - assert.True(t, *b.Config.Presets.SourceLinkedDeployment) -} - -func TestSourceLinkedDeploymentDisabled(t *testing.T) { - b, diags := processSourceLinkedBundle(t, false) - require.NoError(t, diags.Error()) - assert.False(t, *b.Config.Presets.SourceLinkedDeployment) -} - -func processSourceLinkedBundle(t *testing.T, presetEnabled bool) (*bundle.Bundle, diag.Diagnostics) { - if runtime.GOOS == "windows" { - t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace") - } - - b := mockBundle(config.Development) - - workspacePath := "/Workspace/lennart@company.com/" - b.SyncRootPath = workspacePath - b.Config.Presets.SourceLinkedDeployment = &presetEnabled - - ctx := dbr.MockRuntime(context.Background(), true) - m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) - diags := bundle.Apply(ctx, b, m) - return b, diags -} diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go index e074c2b84..7ad3dfd8d 100644 --- a/bundle/config/mutator/resolve_variable_references.go +++ b/bundle/config/mutator/resolve_variable_references.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" @@ -15,7 +16,7 @@ import ( type resolveVariableReferences struct { prefixes []string pattern dyn.Pattern - lookupFn func(dyn.Value, dyn.Path) (dyn.Value, error) + lookupFn func(dyn.Value, dyn.Path, *bundle.Bundle) (dyn.Value, error) skipFn func(dyn.Value) bool } @@ -44,16 +45,21 @@ func ResolveVariableReferencesInComplexVariables() bundle.Mutator { } } -func lookup(v dyn.Value, path dyn.Path) (dyn.Value, error) { +func lookup(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) { + if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { + if path.String() == "workspace.file_path" { + return dyn.V(b.SyncRootPath), nil + } + } // Future opportunity: if we lookup this path in both the given root // and the synthesized root, we know if it was explicitly set or implied to be empty. // Then we can emit a warning if it was not explicitly set. return dyn.GetByPath(v, path) } -func lookupForComplexVariables(v dyn.Value, path dyn.Path) (dyn.Value, error) { +func lookupForComplexVariables(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) { if path[0].Key() != "variables" { - return lookup(v, path) + return lookup(v, path, b) } varV, err := dyn.GetByPath(v, path[:len(path)-1]) @@ -71,7 +77,7 @@ func lookupForComplexVariables(v dyn.Value, path dyn.Path) (dyn.Value, error) { return dyn.InvalidValue, errors.New("complex variables cannot contain references to another complex variables") } - return lookup(v, path) + return lookup(v, path, b) } func skipResolvingInNonComplexVariables(v dyn.Value) bool { @@ -83,9 +89,9 @@ func skipResolvingInNonComplexVariables(v dyn.Value) bool { } } -func lookupForVariables(v dyn.Value, path dyn.Path) (dyn.Value, error) { +func lookupForVariables(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) { if path[0].Key() != "variables" { - return lookup(v, path) + return lookup(v, path, b) } varV, err := dyn.GetByPath(v, path[:len(path)-1]) @@ -103,7 +109,7 @@ func lookupForVariables(v dyn.Value, path dyn.Path) (dyn.Value, error) { return dyn.InvalidValue, errors.New("lookup variables cannot contain references to another lookup variables") } - return lookup(v, path) + return lookup(v, path, b) } func (*resolveVariableReferences) Name() string { @@ -125,6 +131,7 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle) varPath := dyn.NewPath(dyn.Key("var")) var diags diag.Diagnostics + err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { // Synthesize a copy of the root that has all fields that are present in the type // but not set in the dynamic value set to their corresponding empty value. @@ -167,7 +174,7 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle) if m.skipFn != nil && m.skipFn(v) { return dyn.InvalidValue, dynvar.ErrSkipResolution } - return m.lookupFn(normalized, path) + return m.lookupFn(normalized, path, b) } } diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go index 07972ecf4..fcad3180e 100644 --- a/bundle/config/mutator/resolve_variable_references_test.go +++ b/bundle/config/mutator/resolve_variable_references_test.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -434,3 +435,57 @@ func TestResolveComplexVariableWithVarReference(t *testing.T) { require.NoError(t, diags.Error()) require.Equal(t, "cicd_template==1.0.0", b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].Libraries[0].Pypi.Package) } + +func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) { + testCases := []struct { + enabled bool + assert func(t *testing.T, b *bundle.Bundle) + }{ + { + true, + func(t *testing.T, b *bundle.Bundle) { + // Variables that use workspace file path should have SyncRootValue during resolution phase + require.Equal(t, "sync/root/path", b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Configuration["source"]) + + // The file path itself should remain the same + require.Equal(t, "file/path", b.Config.Workspace.FilePath) + }, + }, + { + false, + func(t *testing.T, b *bundle.Bundle) { + require.Equal(t, "file/path", b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Configuration["source"]) + require.Equal(t, "file/path", b.Config.Workspace.FilePath) + }, + }, + } + + for _, testCase := range testCases { + b := &bundle.Bundle{ + SyncRootPath: "sync/root/path", + Config: config.Root{ + Presets: config.Presets{ + SourceLinkedDeployment: &testCase.enabled, + }, + Workspace: config.Workspace{ + FilePath: "file/path", + }, + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "pipeline1": { + PipelineSpec: &pipelines.PipelineSpec{ + Configuration: map[string]string{ + "source": "${workspace.file_path}", + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, ResolveVariableReferences("workspace")) + require.NoError(t, diags.Error()) + testCase.assert(t, b) + } +} diff --git a/bundle/deploy/metadata/compute.go b/bundle/deploy/metadata/compute.go index bc8767de4..b47baa6b2 100644 --- a/bundle/deploy/metadata/compute.go +++ b/bundle/deploy/metadata/compute.go @@ -54,5 +54,8 @@ func (m *compute) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { // Set file upload destination of the bundle in metadata b.Metadata.Config.Workspace.FilePath = b.Config.Workspace.FilePath + if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { + b.Metadata.Config.Workspace.FilePath = b.SyncRootPath + } return nil } diff --git a/bundle/deploy/metadata/compute_test.go b/bundle/deploy/metadata/compute_test.go index 2c2c72376..c6fa9bddb 100644 --- a/bundle/deploy/metadata/compute_test.go +++ b/bundle/deploy/metadata/compute_test.go @@ -97,3 +97,24 @@ func TestComputeMetadataMutator(t *testing.T) { assert.Equal(t, expectedMetadata, b.Metadata) } + +func TestComputeMetadataMutatorSourceLinked(t *testing.T) { + syncRootPath := "/Users/shreyas.goenka@databricks.com/source" + enabled := true + b := &bundle.Bundle{ + SyncRootPath: syncRootPath, + Config: config.Root{ + Presets: config.Presets{ + SourceLinkedDeployment: &enabled, + }, + Workspace: config.Workspace{ + FilePath: "/Users/shreyas.goenka@databricks.com/files", + }, + }, + } + + diags := bundle.Apply(context.Background(), b, Compute()) + require.NoError(t, diags.Error()) + + assert.Equal(t, syncRootPath, b.Metadata.Config.Workspace.FilePath) +} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index f0cbc00c2..f7b3cd608 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -41,6 +41,10 @@ func Initialize() bundle.Mutator { mutator.PopulateCurrentUser(), mutator.LoadGitDetails(), + // This mutator needs to be run before variable interpolation and defining default workspace paths + // because it affects how workspace variables are resolved. + mutator.ApplySourceLinkedDeploymentPreset(), + mutator.DefineDefaultWorkspaceRoot(), mutator.ExpandWorkspaceRoot(), mutator.DefineDefaultWorkspacePaths(), @@ -51,6 +55,7 @@ func Initialize() bundle.Mutator { mutator.RewriteWorkspacePrefix(), mutator.SetVariables(), + // Intentionally placed before ResolveVariableReferencesInLookup, ResolveResourceReferences, // ResolveVariableReferencesInComplexVariables and ResolveVariableReferences. // See what is expected in PythonMutatorPhaseInit doc diff --git a/bundle/tests/loader.go b/bundle/tests/loader.go index 5c48d81cb..bb68b3059 100644 --- a/bundle/tests/loader.go +++ b/bundle/tests/loader.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/libs/dbr" "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/experimental/mocks" @@ -66,7 +67,7 @@ func initializeTarget(t *testing.T, path, env string) (*bundle.Bundle, diag.Diag b := load(t, path) configureMock(t, b) - ctx := context.Background() + ctx := dbr.MockRuntime(context.Background(), false) diags := bundle.Apply(ctx, b, bundle.Seq( mutator.SelectTarget(env), phases.Initialize(), diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go index 70c8de12b..7ec1c5ced 100644 --- a/libs/template/renderer_test.go +++ b/libs/template/renderer_test.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/dbr" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/tags" @@ -39,7 +40,7 @@ func assertFilePermissions(t *testing.T, path string, perm fs.FileMode) { } func assertBuiltinTemplateValid(t *testing.T, template string, settings map[string]any, target string, isServicePrincipal, build bool, tempDir string) { - ctx := context.Background() + ctx := dbr.MockRuntime(context.Background(), false) templateFS, err := fs.Sub(builtinTemplates, path.Join("templates", template)) require.NoError(t, err)