steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add volumes and clean room notebooks

This commit is contained in:
Lennart Kats 2024-12-20 20:43:18 +01:00
parent f3923da6a2
commit 6d5cb1d075
No known key found for this signature in database
GPG Key ID: 1EB8B57673197023
2 changed files with 53 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
bundle/config/mutator/apply_presets_catalog_schema.go
View File

@ -211,6 +211,19 @@ func (m *applyPresetsCatalogSchema) Apply(ctx context.Context, b *bundle.Bundle)
}
}
// Volumes
for _, v := range r.Volumes {
if v.CreateVolumeRequestContent == nil {
continue
}
if v.CatalogName == "" {
v.CatalogName = p.Catalog
}
if v.SchemaName == "" {
v.SchemaName = p.Schema
}
}
return diags
}

60
bundle/config/mutator/apply_presets_catalog_schema_test.go
View File

@ -3,6 +3,7 @@ package mutator_test
import (
"context"
"reflect"
"regexp"
"strings"
"testing"
@ -144,6 +145,14 @@ func mockPresetsCatalogSchema() *bundle.Bundle {
},
},
},
Volumes: map[string]*resources.Volume{
"key": {
CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{
CatalogName: "<catalog>",
SchemaName: "<schema>",
},
},
},
},
Presets: config.Presets{
Catalog: "my_catalog",
@ -155,23 +164,24 @@ func mockPresetsCatalogSchema() *bundle.Bundle {
// ignoredFields are fields that should be ignored in the completeness check
var ignoredFields = map[string]string{
"resources.pipelines.key.schema": "schema is still in private preview",
"resources.jobs.key.tasks[0].notebook_task.base_parameters": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].python_wheel_task.named_parameters": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].python_wheel_task.parameters": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].run_job_task.job_parameters": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].spark_jar_task.parameters": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].spark_python_task.parameters": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].spark_submit_task.parameters": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].sql_task.parameters": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].run_job_task.jar_params": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].run_job_task.notebook_params": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].run_job_task.pipeline_params": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].run_job_task.python_named_params": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].run_job_task.python_params": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].run_job_task.spark_submit_params": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].run_job_task.sql_params": "catalog/schema are passed via job parameters",
"resources.pipelines.key.ingestion_definition.objects[0].schema": "schema name is under schema.source_schema/destination_schema",
"resources.pipelines.key.schema": "schema is still in private preview",
"resources.jobs.key.tasks[0].notebook_task.base_parameters": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].python_wheel_task.named_parameters": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].python_wheel_task.parameters": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].run_job_task.job_parameters": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].spark_jar_task.parameters": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].spark_python_task.parameters": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].spark_submit_task.parameters": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].sql_task.parameters": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].run_job_task.jar_params": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].run_job_task.notebook_params": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].run_job_task.pipeline_params": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].run_job_task.python_named_params": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].run_job_task.python_params": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].run_job_task.spark_submit_params": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].run_job_task.sql_params": "catalog/schema are passed via job parameters",
"resources.jobs.key.tasks[0].clean_rooms_notebook_task.notebook_base_parameters": "catalog/schema are properties inside this struct",
"resources.pipelines.key.ingestion_definition.objects[0].schema": "schema name is under schema.source_schema/destination_schema",
"resources.schemas": "schema name of schemas is under resources.schemas.key.Name",
}
@ -236,11 +246,21 @@ func TestApplyPresetsCatalogSchemaCompleteness(t *testing.T) {
// Convert the recordedFields to a set for easier lookup
recordedPaths := make(map[string]struct{})
arrayIndexPattern := regexp.MustCompile(`\[\d+\]`)
for _, field := range recordedFields {
recordedPaths[field.PathString] = struct{}{}
if i := strings.Index(field.PathString, "["); i >= 0 {
// For entries like resources.jobs.key.parameters[1].default, just add resources.jobs.key.parameters
recordedPaths[field.PathString[:i]] = struct{}{}
// Add base paths for any array indices in the path.
// For example, for resources.jobs.key.parameters[0].default we add "resources.jobs.key.parameters
path := field.PathString
path = arrayIndexPattern.ReplaceAllString(path, "[0]")
for {
i := strings.Index(path, "[")
if i < 0 {
break
}
recordedPaths[path[:i]] = struct{}{}
path = path[i+1:]
}
}