Add missing fields

bundle/config/mutator/apply_presets.go

@@ -122,12 +122,61 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 		if t.TriggerPauseStatus == config.Paused {
 			p.Continuous = false
 		}
-		// TODO: add recommendation when catalog is already set?
-		if t.Catalog != "" && p.Catalog == "" && p.Catalog != "hive_metastore" {
-			p.Catalog = t.Catalog
-		}
-		if t.Schema != "" && p.Target == "" {
-			p.Target = t.Schema
+		if t.Catalog != "" && t.Schema != "" {
+			if p.Catalog == "" {
+				p.Catalog = t.Catalog
+			}
+			if p.Target == "" {
+				p.Target = t.Schema
+			}
+			if p.GatewayDefinition != nil {
+				if p.GatewayDefinition.GatewayStorageCatalog == "" {
+					p.GatewayDefinition.GatewayStorageCatalog = t.Catalog
+				}
+				if p.GatewayDefinition.GatewayStorageSchema == "" {
+					p.GatewayDefinition.GatewayStorageSchema = t.Schema
+				}
+			}
+			if p.IngestionDefinition != nil {
+				for _, obj := range p.IngestionDefinition.Objects {
+					if obj.Report != nil {
+						if obj.Report.DestinationCatalog == "" {
+							obj.Report.DestinationCatalog = t.Catalog
+						}
+						if obj.Report.DestinationSchema == "" {
+							obj.Report.DestinationSchema = t.Schema
+						}
+					}
+					if obj.Schema != nil {
+						if obj.Schema.SourceCatalog == "" {
+							obj.Schema.SourceCatalog = t.Catalog
+						}
+						if obj.Schema.SourceSchema == "" {
+							obj.Schema.SourceSchema = t.Schema
+						}
+						if obj.Schema.DestinationCatalog == "" {
+							obj.Schema.DestinationCatalog = t.Catalog
+						}
+						if obj.Schema.DestinationSchema == "" {
+							obj.Schema.DestinationSchema = t.Schema
+						}
+					}
+					if obj.Table != nil {
+						if obj.Table.SourceCatalog == "" {
+							obj.Table.SourceCatalog = t.Catalog
+						}
+						if obj.Table.SourceSchema == "" {
+							obj.Table.SourceSchema = t.Schema
+						}
+						if obj.Table.DestinationCatalog == "" {
+							obj.Table.DestinationCatalog = t.Catalog
+						}
+						if obj.Table.DestinationSchema == "" {
+							obj.Table.DestinationSchema = t.Schema
+						}
+					}
+				}
+			}
 		}
 	}
 
@@ -493,8 +542,7 @@ func recommendCatalogSchemaUsage(b *bundle.Bundle, ctx context.Context, key stri
 
 		if !fileIncludesPattern(ctx, localPath, expected) {
 			diags = diags.Extend(diag.Diagnostics{{
-				Summary: fmt.Sprintf("Use the 'catalog' and 'schema' parameters provided via 'presets.catalog' and 'presets.schema' using\n\n" +
-					fix),
+				Summary:  "Use the 'catalog' and 'schema' parameters provided via 'presets.catalog' and 'presets.schema' using\n\n" + fix,
 				Severity: diag.Recommendation,
 				Locations: []dyn.Location{{
 					File:   localPath,

bundle/config/mutator/apply_presets_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/databricks/databricks-sdk-go/service/pipelines"
 	"github.com/databricks/databricks-sdk-go/service/serving"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -465,8 +466,8 @@ func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
 
 }
 
-func TestApplyPresetsCatalogSchema(t *testing.T) {
-	b := &bundle.Bundle{
+func PresetsMock() *bundle.Bundle {
+	return &bundle.Bundle{
 		Config: config.Root{
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
@@ -477,6 +478,24 @@ func TestApplyPresetsCatalogSchema(t *testing.T) {
 								{Name: "catalog", Default: "<catalog>"},
 								{Name: "schema", Default: "<schema>"},
 							},
+							Tasks: []jobs.Task{
+								{
+									DbtTask: &jobs.DbtTask{
+										Catalog: "<catalog>",
+										Schema:  "<schema>",
+									},
+								},
+								{
+									SparkPythonTask: &jobs.SparkPythonTask{
+										PythonFile: "/file",
+									},
+								},
+								{
+									NotebookTask: &jobs.NotebookTask{
+										NotebookPath: "/notebook",
+									},
+								},
+							},
 						},
 					},
 				},
@@ -486,6 +505,32 @@ func TestApplyPresetsCatalogSchema(t *testing.T) {
 							Name:    "pipeline",
 							Catalog: "<catalog>",
 							Target:  "<schema>",
+							GatewayDefinition: &pipelines.IngestionGatewayPipelineDefinition{
+								GatewayStorageCatalog: "<catalog>",
+								GatewayStorageSchema:  "<schema>",
+							},
+							IngestionDefinition: &pipelines.IngestionPipelineDefinition{
+								Objects: []pipelines.IngestionConfig{
+									{
+										Report: &pipelines.ReportSpec{
+											DestinationCatalog: "<catalog>",
+											DestinationSchema:  "<schema>",
+										},
+										Schema: &pipelines.SchemaSpec{
+											SourceCatalog:      "<catalog>",
+											SourceSchema:       "<schema>",
+											DestinationCatalog: "<catalog>",
+											DestinationSchema:  "<schema>",
+										},
+										Table: &pipelines.TableSpec{
+											SourceCatalog:      "<catalog>",
+											SourceSchema:       "<schema>",
+											DestinationCatalog: "<catalog>",
+											DestinationSchema:  "<schema>",
+										},
+									},
+								},
+							},
 						},
 					},
 				},
@@ -542,6 +587,17 @@ func TestApplyPresetsCatalogSchema(t *testing.T) {
 			},
 		},
 	}
+}
+
+var PresetsIgnoredFields = map[string]string{
+	// Any fields that should be ignored in the completeness check
+	// Example:
+	// "resources.jobs.object.schema_something": "this property doesn't relate to the catalog/schema",
+	"resources.pipelines.key.schema": "schema is still in private preview",
+}
+
+func TestApplyPresetsCatalogSchema(t *testing.T) {
+	b := PresetsMock()
 	b.Config.Presets = config.Presets{
 		Catalog: "my_catalog",
 		Schema:  "my_schema",
@@ -603,16 +659,11 @@ func TestApplyPresetsCatalogSchema(t *testing.T) {
 	for _, f := range recordedFields {
 		val, err := dyn.GetByPath(config, f.Path)
 		require.NoError(t, err, "failed to get path %s", f.Path)
-		require.Equal(t, f.Expected, val.MustString(), "preset value expected for %s based on placeholder %s", f.Path, f.Placeholder)
+		assert.Equal(t, f.Expected, val.MustString(), "preset value expected for %s based on placeholder %s", f.Path, f.Placeholder)
 	}
 
 	// Stage 4: Check completeness
-	ignoredFields := map[string]string{
-		// Any fields that should be ignored in the completeness check
-		// Example:
-		// "resources.jobs.object.schema_something": "this property doesn't relate to the catalog/schema",
-	}
-	checkCompleteness(t, recordedFields, ignoredFields)
+	checkCompleteness(t, recordedFields)
 }
 
 func verifyNoChangesBeforeCleanup(t *testing.T, rootVal dyn.Value, recordedFields []recordedField) {
@@ -626,7 +677,7 @@ func verifyNoChangesBeforeCleanup(t *testing.T, rootVal dyn.Value, recordedField
 	}
 }
 
-func checkCompleteness(t *testing.T, recordedFields []recordedField, ignoredFields map[string]string) {
+func checkCompleteness(t *testing.T, recordedFields []recordedField) {
 	t.Helper()
 
 	// Build a set for recorded fields
@@ -700,7 +751,7 @@ func checkCompleteness(t *testing.T, recordedFields []recordedField, ignoredFiel
 					// Only check if the field is a string
 					if ft.Type.Kind() == reflect.String {
 						if _, recorded := recordedSet[fieldPath]; !recorded {
-							if _, ignored := ignoredFields[fieldPath]; !ignored {
+							if _, ignored := PresetsIgnoredFields[fieldPath]; !ignored {
 								missingFields = append(missingFields, fieldPath)
 							}
 						}
@@ -719,7 +770,7 @@ func checkCompleteness(t *testing.T, recordedFields []recordedField, ignoredFiel
 
 	// Report all missing fields
 	for _, field := range missingFields {
-		t.Errorf("Field %s was not included in the test (should be covered in 'recordedFields' or 'ignoredFields')", field)
+		t.Errorf("Field %s was not included in the catalog/schema presets test. If this is a new field, please add it to PresetsMock or PresetsIgnoredFields and add support for it as appropriate.", field)
 	}
 
 	// Fail the test if there were any missing fields