diff --git a/bundle/config/mutator/apply_presets.go b/bundle/config/mutator/apply_presets.go index 21b52565..e2354130 100644 --- a/bundle/config/mutator/apply_presets.go +++ b/bundle/config/mutator/apply_presets.go @@ -35,8 +35,10 @@ func (m *applyPresets) Name() string { } func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + if d := validatePauseStatus(b); d != nil { - return d + diags = diags.Extend(d) } r := b.Config.Resources @@ -45,7 +47,11 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos tags := toTagArray(t.Tags) // Jobs presets: Prefix, Tags, JobsMaxConcurrentRuns, TriggerPauseStatus - for _, j := range r.Jobs { + for key, j := range r.Jobs { + if j.JobSettings == nil { + diags = diags.Extend(diag.Errorf("job %s is not defined", key)) + continue + } j.Name = prefix + j.Name if j.Tags == nil { j.Tags = make(map[string]string) @@ -77,20 +83,27 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos } // Pipelines presets: Prefix, PipelinesDevelopment - for i := range r.Pipelines { - r.Pipelines[i].Name = prefix + r.Pipelines[i].Name + for key, p := range r.Pipelines { + if p.PipelineSpec == nil { + diags = diags.Extend(diag.Errorf("pipeline %s is not defined", key)) + continue + } + p.Name = prefix + p.Name if config.IsExplicitlyEnabled(t.PipelinesDevelopment) { - r.Pipelines[i].Development = true + p.Development = true } if t.TriggerPauseStatus == config.Paused { - r.Pipelines[i].Continuous = false + p.Continuous = false } - // As of 2024-06, pipelines don't yet support tags } // Models presets: Prefix, Tags - for _, m := range r.Models { + for key, m := range r.Models { + if m.Model == nil { + diags = diags.Extend(diag.Errorf("model %s is not defined", key)) + continue + } m.Name = prefix + m.Name for _, t := range tags { exists := slices.ContainsFunc(m.Tags, func(modelTag ml.ModelTag) bool { @@ -104,7 +117,11 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos } // Experiments presets: Prefix, Tags - for _, e := range r.Experiments { + for key, e := range r.Experiments { + if e.Experiment == nil { + diags = diags.Extend(diag.Errorf("experiment %s is not defined", key)) + continue + } filepath := e.Name dir := path.Dir(filepath) base := path.Base(filepath) @@ -128,40 +145,60 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos } // Model serving endpoint presets: Prefix - for i := range r.ModelServingEndpoints { - r.ModelServingEndpoints[i].Name = normalizePrefix(prefix) + r.ModelServingEndpoints[i].Name + for key, e := range r.ModelServingEndpoints { + if e.CreateServingEndpoint == nil { + diags = diags.Extend(diag.Errorf("model serving endpoint %s is not defined", key)) + continue + } + e.Name = normalizePrefix(prefix) + e.Name // As of 2024-06, model serving endpoints don't yet support tags } // Registered models presets: Prefix - for i := range r.RegisteredModels { - r.RegisteredModels[i].Name = normalizePrefix(prefix) + r.RegisteredModels[i].Name + for key, m := range r.RegisteredModels { + if m.CreateRegisteredModelRequest == nil { + diags = diags.Extend(diag.Errorf("registered model %s is not defined", key)) + continue + } + m.Name = normalizePrefix(prefix) + m.Name // As of 2024-06, registered models don't yet support tags } - // Quality monitors presets: Prefix + // Quality monitors presets: Schedule if t.TriggerPauseStatus == config.Paused { - for i := range r.QualityMonitors { + for key, q := range r.QualityMonitors { + if q.CreateMonitor == nil { + diags = diags.Extend(diag.Errorf("quality monitor %s is not defined", key)) + continue + } // Remove all schedules from monitors, since they don't support pausing/unpausing. // Quality monitors might support the "pause" property in the future, so at the // CLI level we do respect that property if it is set to "unpaused." - if r.QualityMonitors[i].Schedule != nil && r.QualityMonitors[i].Schedule.PauseStatus != catalog.MonitorCronSchedulePauseStatusUnpaused { - r.QualityMonitors[i].Schedule = nil + if q.Schedule != nil && q.Schedule.PauseStatus != catalog.MonitorCronSchedulePauseStatusUnpaused { + q.Schedule = nil } } } // Schemas: Prefix - for i := range r.Schemas { - r.Schemas[i].Name = normalizePrefix(prefix) + r.Schemas[i].Name + for key, s := range r.Schemas { + if s.CreateSchema == nil { + diags = diags.Extend(diag.Errorf("schema %s is not defined", key)) + continue + } + s.Name = normalizePrefix(prefix) + s.Name // HTTP API for schemas doesn't yet support tags. It's only supported in // the Databricks UI and via the SQL API. } // Clusters: Prefix, Tags - for _, c := range r.Clusters { + for key, c := range r.Clusters { + if c.ClusterSpec == nil { + diags = diags.Extend(diag.Errorf("cluster %s is not defined", key)) + continue + } c.ClusterName = prefix + c.ClusterName if c.CustomTags == nil { c.CustomTags = make(map[string]string) @@ -180,7 +217,7 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos r.Dashboards[i].DisplayName = prefix + r.Dashboards[i].DisplayName } - return nil + return diags } func validatePauseStatus(b *bundle.Bundle) diag.Diagnostics { diff --git a/bundle/config/mutator/apply_presets_test.go b/bundle/config/mutator/apply_presets_test.go index ab2478ae..24295da4 100644 --- a/bundle/config/mutator/apply_presets_test.go +++ b/bundle/config/mutator/apply_presets_test.go @@ -251,3 +251,116 @@ func TestApplyPresetsJobsMaxConcurrentRuns(t *testing.T) { }) } } + +func TestApplyPresetsPrefixWithoutJobSettings(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job1": {}, // no jobsettings inside + }, + }, + Presets: config.Presets{ + NamePrefix: "prefix-", + }, + }, + } + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.ApplyPresets()) + + require.ErrorContains(t, diags.Error(), "job job1 is not defined") +} + +func TestApplyPresetsResourceNotDefined(t *testing.T) { + tests := []struct { + resources config.Resources + error string + }{ + { + resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job1": {}, // no jobsettings inside + }, + }, + error: "job job1 is not defined", + }, + { + resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "pipeline1": {}, // no pipelinespec inside + }, + }, + error: "pipeline pipeline1 is not defined", + }, + { + resources: config.Resources{ + Models: map[string]*resources.MlflowModel{ + "model1": {}, // no model inside + }, + }, + error: "model model1 is not defined", + }, + { + resources: config.Resources{ + Experiments: map[string]*resources.MlflowExperiment{ + "experiment1": {}, // no experiment inside + }, + }, + error: "experiment experiment1 is not defined", + }, + { + resources: config.Resources{ + ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ + "endpoint1": {}, // no CreateServingEndpoint inside + }, + RegisteredModels: map[string]*resources.RegisteredModel{ + "model1": {}, // no CreateRegisteredModelRequest inside + }, + }, + error: "model serving endpoint endpoint1 is not defined", + }, + { + resources: config.Resources{ + QualityMonitors: map[string]*resources.QualityMonitor{ + "monitor1": {}, // no CreateMonitor inside + }, + }, + error: "quality monitor monitor1 is not defined", + }, + { + resources: config.Resources{ + Schemas: map[string]*resources.Schema{ + "schema1": {}, // no CreateSchema inside + }, + }, + error: "schema schema1 is not defined", + }, + { + resources: config.Resources{ + Clusters: map[string]*resources.Cluster{ + "cluster1": {}, // no ClusterSpec inside + }, + }, + error: "cluster cluster1 is not defined", + }, + } + + for _, tt := range tests { + t.Run(tt.error, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: tt.resources, + Presets: config.Presets{ + TriggerPauseStatus: config.Paused, + }, + }, + } + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.ApplyPresets()) + + require.ErrorContains(t, diags.Error(), tt.error) + }) + } +} diff --git a/bundle/config/mutator/default_workspace_paths.go b/bundle/config/mutator/default_workspace_paths.go index 71e562b5..02a1ddb3 100644 --- a/bundle/config/mutator/default_workspace_paths.go +++ b/bundle/config/mutator/default_workspace_paths.go @@ -29,6 +29,10 @@ func (m *defineDefaultWorkspacePaths) Apply(ctx context.Context, b *bundle.Bundl b.Config.Workspace.FilePath = path.Join(root, "files") } + if b.Config.Workspace.ResourcePath == "" { + b.Config.Workspace.ResourcePath = path.Join(root, "resources") + } + if b.Config.Workspace.ArtifactPath == "" { b.Config.Workspace.ArtifactPath = path.Join(root, "artifacts") } diff --git a/bundle/config/mutator/default_workspace_paths_test.go b/bundle/config/mutator/default_workspace_paths_test.go index 0ba20ea2..6779c373 100644 --- a/bundle/config/mutator/default_workspace_paths_test.go +++ b/bundle/config/mutator/default_workspace_paths_test.go @@ -22,6 +22,7 @@ func TestDefineDefaultWorkspacePaths(t *testing.T) { diags := bundle.Apply(context.Background(), b, mutator.DefineDefaultWorkspacePaths()) require.NoError(t, diags.Error()) assert.Equal(t, "/files", b.Config.Workspace.FilePath) + assert.Equal(t, "/resources", b.Config.Workspace.ResourcePath) assert.Equal(t, "/artifacts", b.Config.Workspace.ArtifactPath) assert.Equal(t, "/state", b.Config.Workspace.StatePath) } @@ -32,6 +33,7 @@ func TestDefineDefaultWorkspacePathsAlreadySet(t *testing.T) { Workspace: config.Workspace{ RootPath: "/", FilePath: "/foo/bar", + ResourcePath: "/foo/bar", ArtifactPath: "/foo/bar", StatePath: "/foo/bar", }, @@ -40,6 +42,7 @@ func TestDefineDefaultWorkspacePathsAlreadySet(t *testing.T) { diags := bundle.Apply(context.Background(), b, mutator.DefineDefaultWorkspacePaths()) require.NoError(t, diags.Error()) assert.Equal(t, "/foo/bar", b.Config.Workspace.FilePath) + assert.Equal(t, "/foo/bar", b.Config.Workspace.ResourcePath) assert.Equal(t, "/foo/bar", b.Config.Workspace.ArtifactPath) assert.Equal(t, "/foo/bar", b.Config.Workspace.StatePath) } diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go index 70382f05..9944f6ff 100644 --- a/bundle/config/mutator/process_target_mode.go +++ b/bundle/config/mutator/process_target_mode.go @@ -118,15 +118,18 @@ func findNonUserPath(b *bundle.Bundle) string { if b.Config.Workspace.RootPath != "" && !containsName(b.Config.Workspace.RootPath) { return "root_path" } - if b.Config.Workspace.StatePath != "" && !containsName(b.Config.Workspace.StatePath) { - return "state_path" - } if b.Config.Workspace.FilePath != "" && !containsName(b.Config.Workspace.FilePath) { return "file_path" } + if b.Config.Workspace.ResourcePath != "" && !containsName(b.Config.Workspace.ResourcePath) { + return "resource_path" + } if b.Config.Workspace.ArtifactPath != "" && !containsName(b.Config.Workspace.ArtifactPath) { return "artifact_path" } + if b.Config.Workspace.StatePath != "" && !containsName(b.Config.Workspace.StatePath) { + return "state_path" + } return "" } diff --git a/bundle/config/workspace.go b/bundle/config/workspace.go index efc5caa6..878d0783 100644 --- a/bundle/config/workspace.go +++ b/bundle/config/workspace.go @@ -54,6 +54,11 @@ type Workspace struct { // This defaults to "${workspace.root}/files". FilePath string `json:"file_path,omitempty"` + // Remote workspace path for resources with a presence in the workspace. + // These are kept outside [FilePath] to avoid potential naming collisions. + // This defaults to "${workspace.root}/resources". + ResourcePath string `json:"resource_path,omitempty"` + // Remote workspace path for build artifacts. // This defaults to "${workspace.root}/artifacts". ArtifactPath string `json:"artifact_path,omitempty"`