From 6d5cb1d07554511e4f82d09f35f0332d95783ebf Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Fri, 20 Dec 2024 20:43:18 +0100 Subject: [PATCH] Add volumes and clean room notebooks --- .../mutator/apply_presets_catalog_schema.go | 13 ++++ .../apply_presets_catalog_schema_test.go | 60 ++++++++++++------- 2 files changed, 53 insertions(+), 20 deletions(-) diff --git a/bundle/config/mutator/apply_presets_catalog_schema.go b/bundle/config/mutator/apply_presets_catalog_schema.go index 7326c9aea..ad2834444 100644 --- a/bundle/config/mutator/apply_presets_catalog_schema.go +++ b/bundle/config/mutator/apply_presets_catalog_schema.go @@ -211,6 +211,19 @@ func (m *applyPresetsCatalogSchema) Apply(ctx context.Context, b *bundle.Bundle) } } + // Volumes + for _, v := range r.Volumes { + if v.CreateVolumeRequestContent == nil { + continue + } + if v.CatalogName == "" { + v.CatalogName = p.Catalog + } + if v.SchemaName == "" { + v.SchemaName = p.Schema + } + } + return diags } diff --git a/bundle/config/mutator/apply_presets_catalog_schema_test.go b/bundle/config/mutator/apply_presets_catalog_schema_test.go index d0f9975fc..5230b205d 100644 --- a/bundle/config/mutator/apply_presets_catalog_schema_test.go +++ b/bundle/config/mutator/apply_presets_catalog_schema_test.go @@ -3,6 +3,7 @@ package mutator_test import ( "context" "reflect" + "regexp" "strings" "testing" @@ -144,6 +145,14 @@ func mockPresetsCatalogSchema() *bundle.Bundle { }, }, }, + Volumes: map[string]*resources.Volume{ + "key": { + CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{ + CatalogName: "", + SchemaName: "", + }, + }, + }, }, Presets: config.Presets{ Catalog: "my_catalog", @@ -155,23 +164,24 @@ func mockPresetsCatalogSchema() *bundle.Bundle { // ignoredFields are fields that should be ignored in the completeness check var ignoredFields = map[string]string{ - "resources.pipelines.key.schema": "schema is still in private preview", - "resources.jobs.key.tasks[0].notebook_task.base_parameters": "catalog/schema are passed via job parameters", - "resources.jobs.key.tasks[0].python_wheel_task.named_parameters": "catalog/schema are passed via job parameters", - "resources.jobs.key.tasks[0].python_wheel_task.parameters": "catalog/schema are passed via job parameters", - "resources.jobs.key.tasks[0].run_job_task.job_parameters": "catalog/schema are passed via job parameters", - "resources.jobs.key.tasks[0].spark_jar_task.parameters": "catalog/schema are passed via job parameters", - "resources.jobs.key.tasks[0].spark_python_task.parameters": "catalog/schema are passed via job parameters", - "resources.jobs.key.tasks[0].spark_submit_task.parameters": "catalog/schema are passed via job parameters", - "resources.jobs.key.tasks[0].sql_task.parameters": "catalog/schema are passed via job parameters", - "resources.jobs.key.tasks[0].run_job_task.jar_params": "catalog/schema are passed via job parameters", - "resources.jobs.key.tasks[0].run_job_task.notebook_params": "catalog/schema are passed via job parameters", - "resources.jobs.key.tasks[0].run_job_task.pipeline_params": "catalog/schema are passed via job parameters", - "resources.jobs.key.tasks[0].run_job_task.python_named_params": "catalog/schema are passed via job parameters", - "resources.jobs.key.tasks[0].run_job_task.python_params": "catalog/schema are passed via job parameters", - "resources.jobs.key.tasks[0].run_job_task.spark_submit_params": "catalog/schema are passed via job parameters", - "resources.jobs.key.tasks[0].run_job_task.sql_params": "catalog/schema are passed via job parameters", - "resources.pipelines.key.ingestion_definition.objects[0].schema": "schema name is under schema.source_schema/destination_schema", + "resources.pipelines.key.schema": "schema is still in private preview", + "resources.jobs.key.tasks[0].notebook_task.base_parameters": "catalog/schema are passed via job parameters", + "resources.jobs.key.tasks[0].python_wheel_task.named_parameters": "catalog/schema are passed via job parameters", + "resources.jobs.key.tasks[0].python_wheel_task.parameters": "catalog/schema are passed via job parameters", + "resources.jobs.key.tasks[0].run_job_task.job_parameters": "catalog/schema are passed via job parameters", + "resources.jobs.key.tasks[0].spark_jar_task.parameters": "catalog/schema are passed via job parameters", + "resources.jobs.key.tasks[0].spark_python_task.parameters": "catalog/schema are passed via job parameters", + "resources.jobs.key.tasks[0].spark_submit_task.parameters": "catalog/schema are passed via job parameters", + "resources.jobs.key.tasks[0].sql_task.parameters": "catalog/schema are passed via job parameters", + "resources.jobs.key.tasks[0].run_job_task.jar_params": "catalog/schema are passed via job parameters", + "resources.jobs.key.tasks[0].run_job_task.notebook_params": "catalog/schema are passed via job parameters", + "resources.jobs.key.tasks[0].run_job_task.pipeline_params": "catalog/schema are passed via job parameters", + "resources.jobs.key.tasks[0].run_job_task.python_named_params": "catalog/schema are passed via job parameters", + "resources.jobs.key.tasks[0].run_job_task.python_params": "catalog/schema are passed via job parameters", + "resources.jobs.key.tasks[0].run_job_task.spark_submit_params": "catalog/schema are passed via job parameters", + "resources.jobs.key.tasks[0].run_job_task.sql_params": "catalog/schema are passed via job parameters", + "resources.jobs.key.tasks[0].clean_rooms_notebook_task.notebook_base_parameters": "catalog/schema are properties inside this struct", + "resources.pipelines.key.ingestion_definition.objects[0].schema": "schema name is under schema.source_schema/destination_schema", "resources.schemas": "schema name of schemas is under resources.schemas.key.Name", } @@ -236,11 +246,21 @@ func TestApplyPresetsCatalogSchemaCompleteness(t *testing.T) { // Convert the recordedFields to a set for easier lookup recordedPaths := make(map[string]struct{}) + arrayIndexPattern := regexp.MustCompile(`\[\d+\]`) for _, field := range recordedFields { recordedPaths[field.PathString] = struct{}{} - if i := strings.Index(field.PathString, "["); i >= 0 { - // For entries like resources.jobs.key.parameters[1].default, just add resources.jobs.key.parameters - recordedPaths[field.PathString[:i]] = struct{}{} + + // Add base paths for any array indices in the path. + // For example, for resources.jobs.key.parameters[0].default we add "resources.jobs.key.parameters + path := field.PathString + path = arrayIndexPattern.ReplaceAllString(path, "[0]") + for { + i := strings.Index(path, "[") + if i < 0 { + break + } + recordedPaths[path[:i]] = struct{}{} + path = path[i+1:] } }