diff --git a/bundle/config/mutator/apply_presets.go b/bundle/config/mutator/apply_presets.go index 7cfbfd851..8c6415b35 100644 --- a/bundle/config/mutator/apply_presets.go +++ b/bundle/config/mutator/apply_presets.go @@ -122,12 +122,61 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos if t.TriggerPauseStatus == config.Paused { p.Continuous = false } - // TODO: add recommendation when catalog is already set? - if t.Catalog != "" && p.Catalog == "" && p.Catalog != "hive_metastore" { - p.Catalog = t.Catalog - } - if t.Schema != "" && p.Target == "" { - p.Target = t.Schema + if t.Catalog != "" && t.Schema != "" { + if p.Catalog == "" { + p.Catalog = t.Catalog + } + if p.Target == "" { + p.Target = t.Schema + } + if p.GatewayDefinition != nil { + if p.GatewayDefinition.GatewayStorageCatalog == "" { + p.GatewayDefinition.GatewayStorageCatalog = t.Catalog + } + if p.GatewayDefinition.GatewayStorageSchema == "" { + p.GatewayDefinition.GatewayStorageSchema = t.Schema + } + } + if p.IngestionDefinition != nil { + for _, obj := range p.IngestionDefinition.Objects { + if obj.Report != nil { + if obj.Report.DestinationCatalog == "" { + obj.Report.DestinationCatalog = t.Catalog + } + if obj.Report.DestinationSchema == "" { + obj.Report.DestinationSchema = t.Schema + } + } + if obj.Schema != nil { + if obj.Schema.SourceCatalog == "" { + obj.Schema.SourceCatalog = t.Catalog + } + if obj.Schema.SourceSchema == "" { + obj.Schema.SourceSchema = t.Schema + } + if obj.Schema.DestinationCatalog == "" { + obj.Schema.DestinationCatalog = t.Catalog + } + if obj.Schema.DestinationSchema == "" { + obj.Schema.DestinationSchema = t.Schema + } + } + if obj.Table != nil { + if obj.Table.SourceCatalog == "" { + obj.Table.SourceCatalog = t.Catalog + } + if obj.Table.SourceSchema == "" { + obj.Table.SourceSchema = t.Schema + } + if obj.Table.DestinationCatalog == "" { + obj.Table.DestinationCatalog = t.Catalog + } + if obj.Table.DestinationSchema == "" { + obj.Table.DestinationSchema = t.Schema + } + } + } + } } } @@ -493,8 +542,7 @@ func recommendCatalogSchemaUsage(b *bundle.Bundle, ctx context.Context, key stri if !fileIncludesPattern(ctx, localPath, expected) { diags = diags.Extend(diag.Diagnostics{{ - Summary: fmt.Sprintf("Use the 'catalog' and 'schema' parameters provided via 'presets.catalog' and 'presets.schema' using\n\n" + - fix), + Summary: "Use the 'catalog' and 'schema' parameters provided via 'presets.catalog' and 'presets.schema' using\n\n" + fix, Severity: diag.Recommendation, Locations: []dyn.Location{{ File: localPath, diff --git a/bundle/config/mutator/apply_presets_test.go b/bundle/config/mutator/apply_presets_test.go index 5a05115c4..c94f510dd 100644 --- a/bundle/config/mutator/apply_presets_test.go +++ b/bundle/config/mutator/apply_presets_test.go @@ -18,6 +18,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -465,8 +466,8 @@ func TestApplyPresetsSourceLinkedDeployment(t *testing.T) { } -func TestApplyPresetsCatalogSchema(t *testing.T) { - b := &bundle.Bundle{ +func PresetsMock() *bundle.Bundle { + return &bundle.Bundle{ Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -477,6 +478,24 @@ func TestApplyPresetsCatalogSchema(t *testing.T) { {Name: "catalog", Default: ""}, {Name: "schema", Default: ""}, }, + Tasks: []jobs.Task{ + { + DbtTask: &jobs.DbtTask{ + Catalog: "", + Schema: "", + }, + }, + { + SparkPythonTask: &jobs.SparkPythonTask{ + PythonFile: "/file", + }, + }, + { + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "/notebook", + }, + }, + }, }, }, }, @@ -486,6 +505,32 @@ func TestApplyPresetsCatalogSchema(t *testing.T) { Name: "pipeline", Catalog: "", Target: "", + GatewayDefinition: &pipelines.IngestionGatewayPipelineDefinition{ + GatewayStorageCatalog: "", + GatewayStorageSchema: "", + }, + IngestionDefinition: &pipelines.IngestionPipelineDefinition{ + Objects: []pipelines.IngestionConfig{ + { + Report: &pipelines.ReportSpec{ + DestinationCatalog: "", + DestinationSchema: "", + }, + Schema: &pipelines.SchemaSpec{ + SourceCatalog: "", + SourceSchema: "", + DestinationCatalog: "", + DestinationSchema: "", + }, + Table: &pipelines.TableSpec{ + SourceCatalog: "", + SourceSchema: "", + DestinationCatalog: "", + DestinationSchema: "", + }, + }, + }, + }, }, }, }, @@ -542,6 +587,17 @@ func TestApplyPresetsCatalogSchema(t *testing.T) { }, }, } +} + +var PresetsIgnoredFields = map[string]string{ + // Any fields that should be ignored in the completeness check + // Example: + // "resources.jobs.object.schema_something": "this property doesn't relate to the catalog/schema", + "resources.pipelines.key.schema": "schema is still in private preview", +} + +func TestApplyPresetsCatalogSchema(t *testing.T) { + b := PresetsMock() b.Config.Presets = config.Presets{ Catalog: "my_catalog", Schema: "my_schema", @@ -603,16 +659,11 @@ func TestApplyPresetsCatalogSchema(t *testing.T) { for _, f := range recordedFields { val, err := dyn.GetByPath(config, f.Path) require.NoError(t, err, "failed to get path %s", f.Path) - require.Equal(t, f.Expected, val.MustString(), "preset value expected for %s based on placeholder %s", f.Path, f.Placeholder) + assert.Equal(t, f.Expected, val.MustString(), "preset value expected for %s based on placeholder %s", f.Path, f.Placeholder) } // Stage 4: Check completeness - ignoredFields := map[string]string{ - // Any fields that should be ignored in the completeness check - // Example: - // "resources.jobs.object.schema_something": "this property doesn't relate to the catalog/schema", - } - checkCompleteness(t, recordedFields, ignoredFields) + checkCompleteness(t, recordedFields) } func verifyNoChangesBeforeCleanup(t *testing.T, rootVal dyn.Value, recordedFields []recordedField) { @@ -626,7 +677,7 @@ func verifyNoChangesBeforeCleanup(t *testing.T, rootVal dyn.Value, recordedField } } -func checkCompleteness(t *testing.T, recordedFields []recordedField, ignoredFields map[string]string) { +func checkCompleteness(t *testing.T, recordedFields []recordedField) { t.Helper() // Build a set for recorded fields @@ -700,7 +751,7 @@ func checkCompleteness(t *testing.T, recordedFields []recordedField, ignoredFiel // Only check if the field is a string if ft.Type.Kind() == reflect.String { if _, recorded := recordedSet[fieldPath]; !recorded { - if _, ignored := ignoredFields[fieldPath]; !ignored { + if _, ignored := PresetsIgnoredFields[fieldPath]; !ignored { missingFields = append(missingFields, fieldPath) } } @@ -719,7 +770,7 @@ func checkCompleteness(t *testing.T, recordedFields []recordedField, ignoredFiel // Report all missing fields for _, field := range missingFields { - t.Errorf("Field %s was not included in the test (should be covered in 'recordedFields' or 'ignoredFields')", field) + t.Errorf("Field %s was not included in the catalog/schema presets test. If this is a new field, please add it to PresetsMock or PresetsIgnoredFields and add support for it as appropriate.", field) } // Fail the test if there were any missing fields