From f70ec359dcee7da682702c86fb7e585ae32027fc Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 16 Feb 2024 21:54:38 +0100 Subject: [PATCH] Use `dyn.Value` as input to generating Terraform JSON (#1218) ## Changes This builds on #1098 and uses the `dyn.Value` representation of the bundle configuration to generate the Terraform JSON definition of resources in the bundle. The existing code (in `BundleToTerraform`) was not great and in an effort to slightly improve this, I added a package `tfdyn` that includes dedicated files for each resource type. Every resource type has its own conversion type that takes the `dyn.Value` of the bundle-side resource and converts it into Terraform resources (e.g. a job and optionally its permissions). Because we now use a `dyn.Value` as input, we can represent and emit zero-values that have so far been omitted. For example, setting `num_workers: 0` in your bundle configuration now propagates all the way to the Terraform JSON definition. ## Tests * Unit tests for every converter. I reused the test inputs from `convert_test.go`. * Equivalence tests in every existing test case checks that the resulting JSON is identical. * I manually compared the TF JSON file generated by the CLI from the main branch and from this PR on all of our bundles and bundle examples (internal and external) and found the output doesn't change (with the exception of the odd zero-value being included by the version in this PR). --- bundle/deploy/terraform/convert.go | 61 +++++++++ bundle/deploy/terraform/convert_test.go | 64 +++++++++ bundle/deploy/terraform/tfdyn/convert.go | 23 ++++ .../terraform/tfdyn/convert_experiment.go | 45 ++++++ .../tfdyn/convert_experiment_test.go | 52 +++++++ .../deploy/terraform/tfdyn/convert_grants.go | 39 ++++++ .../terraform/tfdyn/convert_grants_test.go | 71 ++++++++++ bundle/deploy/terraform/tfdyn/convert_job.go | 87 ++++++++++++ .../terraform/tfdyn/convert_job_test.go | 129 ++++++++++++++++++ .../deploy/terraform/tfdyn/convert_model.go | 45 ++++++ .../tfdyn/convert_model_serving_endpoint.go | 45 ++++++ .../convert_model_serving_endpoint_test.go | 88 ++++++++++++ .../terraform/tfdyn/convert_model_test.go | 74 ++++++++++ .../terraform/tfdyn/convert_permissions.go | 32 +++++ .../tfdyn/convert_permissions_test.go | 85 ++++++++++++ .../terraform/tfdyn/convert_pipeline.go | 55 ++++++++ .../terraform/tfdyn/convert_pipeline_test.go | 128 +++++++++++++++++ .../tfdyn/convert_registered_model.go | 45 ++++++ .../tfdyn/convert_registered_model_test.go | 58 ++++++++ bundle/deploy/terraform/tfdyn/rename_keys.go | 46 +++++++ bundle/deploy/terraform/write.go | 12 +- libs/dyn/path.go | 8 ++ 22 files changed, 1291 insertions(+), 1 deletion(-) create mode 100644 bundle/deploy/terraform/tfdyn/convert.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_experiment.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_experiment_test.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_grants.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_grants_test.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_job.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_job_test.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_model.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_model_test.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_permissions.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_permissions_test.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_pipeline.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_pipeline_test.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_registered_model.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_registered_model_test.go create mode 100644 bundle/deploy/terraform/tfdyn/rename_keys.go diff --git a/bundle/deploy/terraform/convert.go b/bundle/deploy/terraform/convert.go index 6723caee..f2fb77e1 100644 --- a/bundle/deploy/terraform/convert.go +++ b/bundle/deploy/terraform/convert.go @@ -1,13 +1,16 @@ package terraform import ( + "context" "encoding/json" "fmt" "reflect" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deploy/terraform/tfdyn" "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" tfjson "github.com/hashicorp/terraform-json" ) @@ -228,6 +231,64 @@ func BundleToTerraform(config *config.Root) *schema.Root { return tfroot } +// BundleToTerraformWithDynValue converts resources in a bundle configuration +// to the equivalent Terraform JSON representation. +func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema.Root, error) { + tfroot := schema.NewRoot() + tfroot.Provider = schema.NewProviders() + + // Convert each resource in the bundle to the equivalent Terraform representation. + resources, err := dyn.Get(root, "resources") + if err != nil { + // If the resources key is missing, return an empty root. + if dyn.IsNoSuchKeyError(err) { + return tfroot, nil + } + return nil, err + } + + tfroot.Resource = schema.NewResources() + + numResources := 0 + _, err = dyn.Walk(resources, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + if len(p) < 2 { + return v, nil + } + + typ := p[0].Key() + key := p[1].Key() + + // Lookup the converter based on the resource type. + c, ok := tfdyn.GetConverter(typ) + if !ok { + return dyn.InvalidValue, fmt.Errorf("no converter for resource type %s", typ) + } + + // Convert resource to Terraform representation. + err := c.Convert(ctx, key, v, tfroot.Resource) + if err != nil { + return dyn.InvalidValue, err + } + + numResources++ + + // Skip traversal of the resource itself. + return v, dyn.ErrSkip + }) + if err != nil { + return nil, err + } + + // We explicitly set "resource" to nil to omit it from a JSON encoding. + // This is required because the terraform CLI requires >= 1 resources defined + // if the "resource" property is used in a .tf.json file. + if numResources == 0 { + tfroot.Resource = nil + } + + return tfroot, nil +} + func TerraformToBundle(state *tfjson.State, config *config.Root) error { if state.Values != nil && state.Values.RootModule != nil { for _, resource := range state.Values.RootModule.Resources { diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go index afef3708..fa59e092 100644 --- a/bundle/deploy/terraform/convert_test.go +++ b/bundle/deploy/terraform/convert_test.go @@ -1,12 +1,16 @@ package terraform import ( + "context" + "encoding/json" "reflect" "testing" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -65,6 +69,8 @@ func TestBundleToTerraformJob(t *testing.T) { assert.Equal(t, "param1", resource.Parameter[0].Name) assert.Equal(t, "param2", resource.Parameter[1].Name) assert.Nil(t, out.Data) + + bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformJobPermissions(t *testing.T) { @@ -92,6 +98,8 @@ func TestBundleToTerraformJobPermissions(t *testing.T) { assert.Len(t, resource.AccessControl, 1) assert.Equal(t, "jane@doe.com", resource.AccessControl[0].UserName) assert.Equal(t, "CAN_VIEW", resource.AccessControl[0].PermissionLevel) + + bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformJobTaskLibraries(t *testing.T) { @@ -128,6 +136,8 @@ func TestBundleToTerraformJobTaskLibraries(t *testing.T) { require.Len(t, resource.Task, 1) require.Len(t, resource.Task[0].Library, 1) assert.Equal(t, "mlflow", resource.Task[0].Library[0].Pypi.Package) + + bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformPipeline(t *testing.T) { @@ -188,6 +198,8 @@ func TestBundleToTerraformPipeline(t *testing.T) { assert.Equal(t, resource.Notification[1].Alerts, []string{"on-update-failure", "on-flow-failure"}) assert.Equal(t, resource.Notification[1].EmailRecipients, []string{"jane@doe.com", "john@doe.com"}) assert.Nil(t, out.Data) + + bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformPipelinePermissions(t *testing.T) { @@ -215,6 +227,8 @@ func TestBundleToTerraformPipelinePermissions(t *testing.T) { assert.Len(t, resource.AccessControl, 1) assert.Equal(t, "jane@doe.com", resource.AccessControl[0].UserName) assert.Equal(t, "CAN_VIEW", resource.AccessControl[0].PermissionLevel) + + bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformModel(t *testing.T) { @@ -254,10 +268,15 @@ func TestBundleToTerraformModel(t *testing.T) { assert.Equal(t, "k2", resource.Tags[1].Key) assert.Equal(t, "v2", resource.Tags[1].Value) assert.Nil(t, out.Data) + + bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformModelPermissions(t *testing.T) { var src = resources.MlflowModel{ + Model: &ml.Model{ + Name: "name", + }, Permissions: []resources.Permission{ { Level: "CAN_READ", @@ -281,6 +300,8 @@ func TestBundleToTerraformModelPermissions(t *testing.T) { assert.Len(t, resource.AccessControl, 1) assert.Equal(t, "jane@doe.com", resource.AccessControl[0].UserName) assert.Equal(t, "CAN_READ", resource.AccessControl[0].PermissionLevel) + + bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformExperiment(t *testing.T) { @@ -303,10 +324,15 @@ func TestBundleToTerraformExperiment(t *testing.T) { assert.Equal(t, "name", resource.Name) assert.Nil(t, out.Data) + + bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformExperimentPermissions(t *testing.T) { var src = resources.MlflowExperiment{ + Experiment: &ml.Experiment{ + Name: "name", + }, Permissions: []resources.Permission{ { Level: "CAN_READ", @@ -331,6 +357,7 @@ func TestBundleToTerraformExperimentPermissions(t *testing.T) { assert.Equal(t, "jane@doe.com", resource.AccessControl[0].UserName) assert.Equal(t, "CAN_READ", resource.AccessControl[0].PermissionLevel) + bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformModelServing(t *testing.T) { @@ -377,10 +404,15 @@ func TestBundleToTerraformModelServing(t *testing.T) { assert.Equal(t, "model_name-1", resource.Config.TrafficConfig.Routes[0].ServedModelName) assert.Equal(t, 100, resource.Config.TrafficConfig.Routes[0].TrafficPercentage) assert.Nil(t, out.Data) + + bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformModelServingPermissions(t *testing.T) { var src = resources.ModelServingEndpoint{ + CreateServingEndpoint: &serving.CreateServingEndpoint{ + Name: "name", + }, Permissions: []resources.Permission{ { Level: "CAN_VIEW", @@ -405,6 +437,7 @@ func TestBundleToTerraformModelServingPermissions(t *testing.T) { assert.Equal(t, "jane@doe.com", resource.AccessControl[0].UserName) assert.Equal(t, "CAN_VIEW", resource.AccessControl[0].PermissionLevel) + bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformRegisteredModel(t *testing.T) { @@ -433,10 +466,17 @@ func TestBundleToTerraformRegisteredModel(t *testing.T) { assert.Equal(t, "schema", resource.SchemaName) assert.Equal(t, "comment", resource.Comment) assert.Nil(t, out.Data) + + bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformRegisteredModelGrants(t *testing.T) { var src = resources.RegisteredModel{ + CreateRegisteredModelRequest: &catalog.CreateRegisteredModelRequest{ + Name: "name", + CatalogName: "catalog", + SchemaName: "schema", + }, Grants: []resources.Grant{ { Privileges: []string{"EXECUTE"}, @@ -460,6 +500,8 @@ func TestBundleToTerraformRegisteredModelGrants(t *testing.T) { assert.Len(t, resource.Grant, 1) assert.Equal(t, "jane@doe.com", resource.Grant[0].Principal) assert.Equal(t, "EXECUTE", resource.Grant[0].Privileges[0]) + + bundleToTerraformEquivalenceTest(t, &config) } func TestTerraformToBundleEmptyLocalResources(t *testing.T) { @@ -827,3 +869,25 @@ func AssertFullResourceCoverage(t *testing.T, config *config.Root) { } } } + +func assertEqualTerraformRoot(t *testing.T, a, b *schema.Root) { + ba, err := json.Marshal(a) + require.NoError(t, err) + bb, err := json.Marshal(b) + require.NoError(t, err) + assert.JSONEq(t, string(ba), string(bb)) +} + +func bundleToTerraformEquivalenceTest(t *testing.T, config *config.Root) { + t.Run("dyn equivalence", func(t *testing.T) { + tf1 := BundleToTerraform(config) + + vin, err := convert.FromTyped(config, dyn.NilValue) + require.NoError(t, err) + tf2, err := BundleToTerraformWithDynValue(context.Background(), vin) + require.NoError(t, err) + + // Compare roots + assertEqualTerraformRoot(t, tf1, tf2) + }) +} diff --git a/bundle/deploy/terraform/tfdyn/convert.go b/bundle/deploy/terraform/tfdyn/convert.go new file mode 100644 index 00000000..9df4e264 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert.go @@ -0,0 +1,23 @@ +package tfdyn + +import ( + "context" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" +) + +type Converter interface { + Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error +} + +var converters = map[string]Converter{} + +func GetConverter(name string) (Converter, bool) { + c, ok := converters[name] + return c, ok +} + +func registerConverter(name string, c Converter) { + converters[name] = c +} diff --git a/bundle/deploy/terraform/tfdyn/convert_experiment.go b/bundle/deploy/terraform/tfdyn/convert_experiment.go new file mode 100644 index 00000000..0c129181 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_experiment.go @@ -0,0 +1,45 @@ +package tfdyn + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/log" +) + +func convertExperimentResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) { + // Normalize the output value to the target schema. + vout, diags := convert.Normalize(schema.ResourceMlflowExperiment{}, vin) + for _, diag := range diags { + log.Debugf(ctx, "experiment normalization diagnostic: %s", diag.Summary) + } + + return vout, nil +} + +type experimentConverter struct{} + +func (experimentConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error { + vout, err := convertExperimentResource(ctx, vin) + if err != nil { + return err + } + + // Add the converted resource to the output. + out.MlflowExperiment[key] = vout.AsAny() + + // Configure permissions for this resource. + if permissions := convertPermissionsResource(ctx, vin); permissions != nil { + permissions.ExperimentId = fmt.Sprintf("${databricks_mlflow_experiment.%s.id}", key) + out.Permissions["mlflow_experiment_"+key] = permissions + } + + return nil +} + +func init() { + registerConverter("experiments", experimentConverter{}) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_experiment_test.go b/bundle/deploy/terraform/tfdyn/convert_experiment_test.go new file mode 100644 index 00000000..63add436 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_experiment_test.go @@ -0,0 +1,52 @@ +package tfdyn + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertExperiment(t *testing.T) { + var src = resources.MlflowExperiment{ + Experiment: &ml.Experiment{ + Name: "name", + }, + Permissions: []resources.Permission{ + { + Level: "CAN_READ", + UserName: "jane@doe.com", + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + out := schema.NewResources() + err = experimentConverter{}.Convert(ctx, "my_experiment", vin, out) + require.NoError(t, err) + + // Assert equality on the experiment + assert.Equal(t, map[string]any{ + "name": "name", + }, out.MlflowExperiment["my_experiment"]) + + // Assert equality on the permissions + assert.Equal(t, &schema.ResourcePermissions{ + ExperimentId: "${databricks_mlflow_experiment.my_experiment.id}", + AccessControl: []schema.ResourcePermissionsAccessControl{ + { + PermissionLevel: "CAN_READ", + UserName: "jane@doe.com", + }, + }, + }, out.Permissions["mlflow_experiment_my_experiment"]) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_grants.go b/bundle/deploy/terraform/tfdyn/convert_grants.go new file mode 100644 index 00000000..1ddd99dd --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_grants.go @@ -0,0 +1,39 @@ +package tfdyn + +import ( + "context" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" +) + +func convertGrantsResource(ctx context.Context, vin dyn.Value) *schema.ResourceGrants { + grants, ok := vin.Get("grants").AsSequence() + if !ok || len(grants) == 0 { + return nil + } + + resource := &schema.ResourceGrants{} + for _, permission := range grants { + principal, _ := permission.Get("principal").AsString() + v, _ := permission.Get("privileges").AsSequence() + + // Turn privileges into a slice of strings. + var privileges []string + for _, privilege := range v { + str, ok := privilege.AsString() + if !ok { + continue + } + + privileges = append(privileges, str) + } + + resource.Grant = append(resource.Grant, schema.ResourceGrantsGrant{ + Principal: principal, + Privileges: privileges, + }) + } + + return resource +} diff --git a/bundle/deploy/terraform/tfdyn/convert_grants_test.go b/bundle/deploy/terraform/tfdyn/convert_grants_test.go new file mode 100644 index 00000000..a486bc36 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_grants_test.go @@ -0,0 +1,71 @@ +package tfdyn + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertGrants(t *testing.T) { + var src = resources.RegisteredModel{ + Grants: []resources.Grant{ + { + Privileges: []string{"EXECUTE", "FOO"}, + Principal: "jane@doe.com", + }, + { + Privileges: []string{"EXECUTE", "BAR"}, + Principal: "spn", + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + resource := convertGrantsResource(ctx, vin) + require.NotNil(t, resource) + assert.Equal(t, []schema.ResourceGrantsGrant{ + { + Privileges: []string{"EXECUTE", "FOO"}, + Principal: "jane@doe.com", + }, + { + Privileges: []string{"EXECUTE", "BAR"}, + Principal: "spn", + }, + }, resource.Grant) +} + +func TestConvertGrantsNil(t *testing.T) { + var src = resources.RegisteredModel{ + Grants: nil, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + resource := convertGrantsResource(ctx, vin) + assert.Nil(t, resource) +} + +func TestConvertGrantsEmpty(t *testing.T) { + var src = resources.RegisteredModel{ + Grants: []resources.Grant{}, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + resource := convertGrantsResource(ctx, vin) + assert.Nil(t, resource) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_job.go b/bundle/deploy/terraform/tfdyn/convert_job.go new file mode 100644 index 00000000..b488df15 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_job.go @@ -0,0 +1,87 @@ +package tfdyn + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/service/jobs" +) + +func convertJobResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) { + // Normalize the input value to the underlying job schema. + // This removes superfluous keys and adapts the input to the expected schema. + vin, diags := convert.Normalize(jobs.JobSettings{}, vin) + for _, diag := range diags { + log.Debugf(ctx, "job normalization diagnostic: %s", diag.Summary) + } + + // Modify top-level keys. + vout, err := renameKeys(vin, map[string]string{ + "tasks": "task", + "job_clusters": "job_cluster", + "parameters": "parameter", + }) + if err != nil { + return dyn.InvalidValue, err + } + + // Modify keys in the "git_source" block + vout, err = dyn.Map(vout, "git_source", func(v dyn.Value) (dyn.Value, error) { + return renameKeys(v, map[string]string{ + "git_branch": "branch", + "git_commit": "commit", + "git_provider": "provider", + "git_tag": "tag", + "git_url": "url", + }) + }) + if err != nil { + return dyn.InvalidValue, err + } + + // Modify keys in the "task" blocks + vout, err = dyn.Map(vout, "task", dyn.Foreach(func(v dyn.Value) (dyn.Value, error) { + return renameKeys(v, map[string]string{ + "libraries": "library", + }) + })) + if err != nil { + return dyn.InvalidValue, err + } + + // Normalize the output value to the target schema. + vout, diags = convert.Normalize(schema.ResourceJob{}, vout) + for _, diag := range diags { + log.Debugf(ctx, "job normalization diagnostic: %s", diag.Summary) + } + + return vout, err +} + +type jobConverter struct{} + +func (jobConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error { + vout, err := convertJobResource(ctx, vin) + if err != nil { + return err + } + + // Add the converted resource to the output. + out.Job[key] = vout.AsAny() + + // Configure permissions for this resource. + if permissions := convertPermissionsResource(ctx, vin); permissions != nil { + permissions.JobId = fmt.Sprintf("${databricks_job.%s.id}", key) + out.Permissions["job_"+key] = permissions + } + + return nil +} + +func init() { + registerConverter("jobs", jobConverter{}) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_job_test.go b/bundle/deploy/terraform/tfdyn/convert_job_test.go new file mode 100644 index 00000000..4e988b14 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_job_test.go @@ -0,0 +1,129 @@ +package tfdyn + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertJob(t *testing.T) { + var src = resources.Job{ + JobSettings: &jobs.JobSettings{ + Name: "my job", + JobClusters: []jobs.JobCluster{ + { + JobClusterKey: "key", + NewCluster: &compute.ClusterSpec{ + SparkVersion: "10.4.x-scala2.12", + }, + }, + }, + GitSource: &jobs.GitSource{ + GitProvider: jobs.GitProviderGitHub, + GitUrl: "https://github.com/foo/bar", + }, + Parameters: []jobs.JobParameterDefinition{ + { + Name: "param1", + Default: "default1", + }, + { + Name: "param2", + Default: "default2", + }, + }, + Tasks: []jobs.Task{ + { + TaskKey: "task_key", + JobClusterKey: "job_cluster_key", + Libraries: []compute.Library{ + { + Pypi: &compute.PythonPyPiLibrary{ + Package: "package", + }, + }, + { + Whl: "/path/to/my.whl", + }, + }, + }, + }, + }, + Permissions: []resources.Permission{ + { + Level: "CAN_VIEW", + UserName: "jane@doe.com", + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + out := schema.NewResources() + err = jobConverter{}.Convert(ctx, "my_job", vin, out) + require.NoError(t, err) + + // Assert equality on the job + assert.Equal(t, map[string]any{ + "name": "my job", + "job_cluster": []any{ + map[string]any{ + "job_cluster_key": "key", + "new_cluster": map[string]any{ + "spark_version": "10.4.x-scala2.12", + }, + }, + }, + "git_source": map[string]any{ + "provider": "gitHub", + "url": "https://github.com/foo/bar", + }, + "parameter": []any{ + map[string]any{ + "name": "param1", + "default": "default1", + }, + map[string]any{ + "name": "param2", + "default": "default2", + }, + }, + "task": []any{ + map[string]any{ + "task_key": "task_key", + "job_cluster_key": "job_cluster_key", + "library": []any{ + map[string]any{ + "pypi": map[string]any{ + "package": "package", + }, + }, + map[string]any{ + "whl": "/path/to/my.whl", + }, + }, + }, + }, + }, out.Job["my_job"]) + + // Assert equality on the permissions + assert.Equal(t, &schema.ResourcePermissions{ + JobId: "${databricks_job.my_job.id}", + AccessControl: []schema.ResourcePermissionsAccessControl{ + { + PermissionLevel: "CAN_VIEW", + UserName: "jane@doe.com", + }, + }, + }, out.Permissions["job_my_job"]) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_model.go b/bundle/deploy/terraform/tfdyn/convert_model.go new file mode 100644 index 00000000..f5d7d489 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_model.go @@ -0,0 +1,45 @@ +package tfdyn + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/log" +) + +func convertModelResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) { + // Normalize the output value to the target schema. + vout, diags := convert.Normalize(schema.ResourceMlflowModel{}, vin) + for _, diag := range diags { + log.Debugf(ctx, "model normalization diagnostic: %s", diag.Summary) + } + + return vout, nil +} + +type modelConverter struct{} + +func (modelConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error { + vout, err := convertModelResource(ctx, vin) + if err != nil { + return err + } + + // Add the converted resource to the output. + out.MlflowModel[key] = vout.AsAny() + + // Configure permissions for this resource. + if permissions := convertPermissionsResource(ctx, vin); permissions != nil { + permissions.RegisteredModelId = fmt.Sprintf("${databricks_mlflow_model.%s.registered_model_id}", key) + out.Permissions["mlflow_model_"+key] = permissions + } + + return nil +} + +func init() { + registerConverter("models", modelConverter{}) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint.go b/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint.go new file mode 100644 index 00000000..b67e4dcc --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint.go @@ -0,0 +1,45 @@ +package tfdyn + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/log" +) + +func convertModelServingEndpointResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) { + // Normalize the output value to the target schema. + vout, diags := convert.Normalize(schema.ResourceModelServing{}, vin) + for _, diag := range diags { + log.Debugf(ctx, "model serving endpoint normalization diagnostic: %s", diag.Summary) + } + + return vout, nil +} + +type modelServingEndpointConverter struct{} + +func (modelServingEndpointConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error { + vout, err := convertModelServingEndpointResource(ctx, vin) + if err != nil { + return err + } + + // Add the converted resource to the output. + out.ModelServing[key] = vout.AsAny() + + // Configure permissions for this resource. + if permissions := convertPermissionsResource(ctx, vin); permissions != nil { + permissions.ServingEndpointId = fmt.Sprintf("${databricks_model_serving.%s.serving_endpoint_id}", key) + out.Permissions["model_serving_"+key] = permissions + } + + return nil +} + +func init() { + registerConverter("model_serving_endpoints", modelServingEndpointConverter{}) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go b/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go new file mode 100644 index 00000000..63b75e9a --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go @@ -0,0 +1,88 @@ +package tfdyn + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertModelServingEndpoint(t *testing.T) { + var src = resources.ModelServingEndpoint{ + CreateServingEndpoint: &serving.CreateServingEndpoint{ + Name: "name", + Config: serving.EndpointCoreConfigInput{ + ServedModels: []serving.ServedModelInput{ + { + ModelName: "model_name", + ModelVersion: "1", + ScaleToZeroEnabled: true, + WorkloadSize: "Small", + }, + }, + TrafficConfig: &serving.TrafficConfig{ + Routes: []serving.Route{ + { + ServedModelName: "model_name-1", + TrafficPercentage: 100, + }, + }, + }, + }, + }, + Permissions: []resources.Permission{ + { + Level: "CAN_VIEW", + UserName: "jane@doe.com", + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + out := schema.NewResources() + err = modelServingEndpointConverter{}.Convert(ctx, "my_model_serving_endpoint", vin, out) + require.NoError(t, err) + + // Assert equality on the model serving endpoint + assert.Equal(t, map[string]any{ + "name": "name", + "config": map[string]any{ + "served_models": []any{ + map[string]any{ + "model_name": "model_name", + "model_version": "1", + "scale_to_zero_enabled": true, + "workload_size": "Small", + }, + }, + "traffic_config": map[string]any{ + "routes": []any{ + map[string]any{ + "served_model_name": "model_name-1", + "traffic_percentage": int64(100), + }, + }, + }, + }, + }, out.ModelServing["my_model_serving_endpoint"]) + + // Assert equality on the permissions + assert.Equal(t, &schema.ResourcePermissions{ + ServingEndpointId: "${databricks_model_serving.my_model_serving_endpoint.serving_endpoint_id}", + AccessControl: []schema.ResourcePermissionsAccessControl{ + { + PermissionLevel: "CAN_VIEW", + UserName: "jane@doe.com", + }, + }, + }, out.Permissions["model_serving_my_model_serving_endpoint"]) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_model_test.go b/bundle/deploy/terraform/tfdyn/convert_model_test.go new file mode 100644 index 00000000..542caa87 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_model_test.go @@ -0,0 +1,74 @@ +package tfdyn + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertModel(t *testing.T) { + var src = resources.MlflowModel{ + Model: &ml.Model{ + Name: "name", + Description: "description", + Tags: []ml.ModelTag{ + { + Key: "k1", + Value: "v1", + }, + { + Key: "k2", + Value: "v2", + }, + }, + }, + Permissions: []resources.Permission{ + { + Level: "CAN_READ", + UserName: "jane@doe.com", + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + out := schema.NewResources() + err = modelConverter{}.Convert(ctx, "my_model", vin, out) + require.NoError(t, err) + + // Assert equality on the model + assert.Equal(t, map[string]any{ + "name": "name", + "description": "description", + "tags": []any{ + map[string]any{ + "key": "k1", + "value": "v1", + }, + map[string]any{ + "key": "k2", + "value": "v2", + }, + }, + }, out.MlflowModel["my_model"]) + + // Assert equality on the permissions + assert.Equal(t, &schema.ResourcePermissions{ + RegisteredModelId: "${databricks_mlflow_model.my_model.registered_model_id}", + AccessControl: []schema.ResourcePermissionsAccessControl{ + { + PermissionLevel: "CAN_READ", + UserName: "jane@doe.com", + }, + }, + }, out.Permissions["mlflow_model_my_model"]) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_permissions.go b/bundle/deploy/terraform/tfdyn/convert_permissions.go new file mode 100644 index 00000000..99e8d297 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_permissions.go @@ -0,0 +1,32 @@ +package tfdyn + +import ( + "context" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" +) + +func convertPermissionsResource(ctx context.Context, vin dyn.Value) *schema.ResourcePermissions { + permissions, ok := vin.Get("permissions").AsSequence() + if !ok || len(permissions) == 0 { + return nil + } + + resource := &schema.ResourcePermissions{} + for _, permission := range permissions { + level, _ := permission.Get("level").AsString() + userName, _ := permission.Get("user_name").AsString() + groupName, _ := permission.Get("group_name").AsString() + servicePrincipalName, _ := permission.Get("service_principal_name").AsString() + + resource.AccessControl = append(resource.AccessControl, schema.ResourcePermissionsAccessControl{ + PermissionLevel: level, + UserName: userName, + GroupName: groupName, + ServicePrincipalName: servicePrincipalName, + }) + } + + return resource +} diff --git a/bundle/deploy/terraform/tfdyn/convert_permissions_test.go b/bundle/deploy/terraform/tfdyn/convert_permissions_test.go new file mode 100644 index 00000000..ba389020 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_permissions_test.go @@ -0,0 +1,85 @@ +package tfdyn + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertPermissions(t *testing.T) { + var src = resources.Job{ + Permissions: []resources.Permission{ + { + Level: "CAN_VIEW", + UserName: "jane@doe.com", + }, + { + Level: "CAN_MANAGE", + GroupName: "special admins", + }, + { + Level: "CAN_RUN", + ServicePrincipalName: "spn", + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + resource := convertPermissionsResource(ctx, vin) + require.NotNil(t, resource) + assert.Equal(t, []schema.ResourcePermissionsAccessControl{ + { + PermissionLevel: "CAN_VIEW", + UserName: "jane@doe.com", + GroupName: "", + ServicePrincipalName: "", + }, + { + PermissionLevel: "CAN_MANAGE", + UserName: "", + GroupName: "special admins", + ServicePrincipalName: "", + }, + { + PermissionLevel: "CAN_RUN", + UserName: "", + GroupName: "", + ServicePrincipalName: "spn", + }, + }, resource.AccessControl) +} + +func TestConvertPermissionsNil(t *testing.T) { + var src = resources.Job{ + Permissions: nil, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + resource := convertPermissionsResource(ctx, vin) + assert.Nil(t, resource) +} + +func TestConvertPermissionsEmpty(t *testing.T) { + var src = resources.Job{ + Permissions: []resources.Permission{}, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + resource := convertPermissionsResource(ctx, vin) + assert.Nil(t, resource) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline.go b/bundle/deploy/terraform/tfdyn/convert_pipeline.go new file mode 100644 index 00000000..ea0c94d6 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_pipeline.go @@ -0,0 +1,55 @@ +package tfdyn + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/log" +) + +func convertPipelineResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) { + // Modify top-level keys. + vout, err := renameKeys(vin, map[string]string{ + "libraries": "library", + "clusters": "cluster", + "notifications": "notification", + }) + if err != nil { + return dyn.InvalidValue, err + } + + // Normalize the output value to the target schema. + vout, diags := convert.Normalize(schema.ResourcePipeline{}, vout) + for _, diag := range diags { + log.Debugf(ctx, "pipeline normalization diagnostic: %s", diag.Summary) + } + + return vout, err +} + +type pipelineConverter struct{} + +func (pipelineConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error { + vout, err := convertPipelineResource(ctx, vin) + if err != nil { + return err + } + + // Add the converted resource to the output. + out.Pipeline[key] = vout.AsAny() + + // Configure permissions for this resource. + if permissions := convertPermissionsResource(ctx, vin); permissions != nil { + permissions.PipelineId = fmt.Sprintf("${databricks_pipeline.%s.id}", key) + out.Permissions["pipeline_"+key] = permissions + } + + return nil +} + +func init() { + registerConverter("pipelines", pipelineConverter{}) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go b/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go new file mode 100644 index 00000000..7010d463 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go @@ -0,0 +1,128 @@ +package tfdyn + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertPipeline(t *testing.T) { + var src = resources.Pipeline{ + PipelineSpec: &pipelines.PipelineSpec{ + Name: "my pipeline", + Libraries: []pipelines.PipelineLibrary{ + { + Notebook: &pipelines.NotebookLibrary{ + Path: "notebook path", + }, + }, + { + File: &pipelines.FileLibrary{ + Path: "file path", + }, + }, + }, + Notifications: []pipelines.Notifications{ + { + Alerts: []string{ + "on-update-fatal-failure", + }, + EmailRecipients: []string{ + "jane@doe.com", + }, + }, + { + Alerts: []string{ + "on-update-failure", + "on-flow-failure", + }, + EmailRecipients: []string{ + "jane@doe.com", + "john@doe.com", + }, + }, + }, + Clusters: []pipelines.PipelineCluster{ + { + Label: "default", + NumWorkers: 1, + }, + }, + }, + Permissions: []resources.Permission{ + { + Level: "CAN_VIEW", + UserName: "jane@doe.com", + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + out := schema.NewResources() + err = pipelineConverter{}.Convert(ctx, "my_pipeline", vin, out) + require.NoError(t, err) + + // Assert equality on the pipeline + assert.Equal(t, map[string]any{ + "name": "my pipeline", + "library": []any{ + map[string]any{ + "notebook": map[string]any{ + "path": "notebook path", + }, + }, + map[string]any{ + "file": map[string]any{ + "path": "file path", + }, + }, + }, + "notification": []any{ + map[string]any{ + "alerts": []any{ + "on-update-fatal-failure", + }, + "email_recipients": []any{ + "jane@doe.com", + }, + }, + map[string]any{ + "alerts": []any{ + "on-update-failure", + "on-flow-failure", + }, + "email_recipients": []any{ + "jane@doe.com", + "john@doe.com", + }, + }, + }, + "cluster": []any{ + map[string]any{ + "label": "default", + "num_workers": int64(1), + }, + }, + }, out.Pipeline["my_pipeline"]) + + // Assert equality on the permissions + assert.Equal(t, &schema.ResourcePermissions{ + PipelineId: "${databricks_pipeline.my_pipeline.id}", + AccessControl: []schema.ResourcePermissionsAccessControl{ + { + PermissionLevel: "CAN_VIEW", + UserName: "jane@doe.com", + }, + }, + }, out.Permissions["pipeline_my_pipeline"]) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_registered_model.go b/bundle/deploy/terraform/tfdyn/convert_registered_model.go new file mode 100644 index 00000000..20aa596f --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_registered_model.go @@ -0,0 +1,45 @@ +package tfdyn + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/log" +) + +func convertRegisteredModelResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) { + // Normalize the output value to the target schema. + vout, diags := convert.Normalize(schema.ResourceRegisteredModel{}, vin) + for _, diag := range diags { + log.Debugf(ctx, "registered model normalization diagnostic: %s", diag.Summary) + } + + return vout, nil +} + +type registeredModelConverter struct{} + +func (registeredModelConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error { + vout, err := convertRegisteredModelResource(ctx, vin) + if err != nil { + return err + } + + // Add the converted resource to the output. + out.RegisteredModel[key] = vout.AsAny() + + // Configure grants for this resource. + if grants := convertGrantsResource(ctx, vin); grants != nil { + grants.Function = fmt.Sprintf("${databricks_registered_model.%s.id}", key) + out.Grants["registered_model_"+key] = grants + } + + return nil +} + +func init() { + registerConverter("registered_models", registeredModelConverter{}) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_registered_model_test.go b/bundle/deploy/terraform/tfdyn/convert_registered_model_test.go new file mode 100644 index 00000000..77096e8d --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_registered_model_test.go @@ -0,0 +1,58 @@ +package tfdyn + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertRegisteredModel(t *testing.T) { + var src = resources.RegisteredModel{ + CreateRegisteredModelRequest: &catalog.CreateRegisteredModelRequest{ + Name: "name", + CatalogName: "catalog", + SchemaName: "schema", + Comment: "comment", + }, + Grants: []resources.Grant{ + { + Privileges: []string{"EXECUTE"}, + Principal: "jane@doe.com", + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + out := schema.NewResources() + err = registeredModelConverter{}.Convert(ctx, "my_registered_model", vin, out) + require.NoError(t, err) + + // Assert equality on the registered model + assert.Equal(t, map[string]any{ + "name": "name", + "catalog_name": "catalog", + "schema_name": "schema", + "comment": "comment", + }, out.RegisteredModel["my_registered_model"]) + + // Assert equality on the grants + assert.Equal(t, &schema.ResourceGrants{ + Function: "${databricks_registered_model.my_registered_model.id}", + Grant: []schema.ResourceGrantsGrant{ + { + Privileges: []string{"EXECUTE"}, + Principal: "jane@doe.com", + }, + }, + }, out.Grants["registered_model_my_registered_model"]) +} diff --git a/bundle/deploy/terraform/tfdyn/rename_keys.go b/bundle/deploy/terraform/tfdyn/rename_keys.go new file mode 100644 index 00000000..a65c9f25 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/rename_keys.go @@ -0,0 +1,46 @@ +package tfdyn + +import ( + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/merge" +) + +// renameKeys renames keys in the given map value. +// +// Terraform resources sometimes use singular names for repeating blocks where the API +// definition uses the plural name. This function can convert between the two. +func renameKeys(v dyn.Value, rename map[string]string) (dyn.Value, error) { + var err error + var acc = dyn.V(map[string]dyn.Value{}) + + nv, err := dyn.Walk(v, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + if len(p) == 0 { + return v, nil + } + + // Check if this key should be renamed. + for oldKey, newKey := range rename { + if p[0].Key() != oldKey { + continue + } + + // Add the new key to the accumulator. + p[0] = dyn.Key(newKey) + acc, err = dyn.SetByPath(acc, p, v) + if err != nil { + return dyn.NilValue, err + } + return dyn.InvalidValue, dyn.ErrDrop + } + + // Pass through all other values. + return v, dyn.ErrSkip + }) + + if err != nil { + return dyn.InvalidValue, err + } + + // Merge the accumulator with the original value. + return merge.Merge(nv, acc) +} diff --git a/bundle/deploy/terraform/write.go b/bundle/deploy/terraform/write.go index b53f9069..3ec1b581 100644 --- a/bundle/deploy/terraform/write.go +++ b/bundle/deploy/terraform/write.go @@ -7,6 +7,8 @@ import ( "path/filepath" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" ) type write struct{} @@ -21,7 +23,15 @@ func (w *write) Apply(ctx context.Context, b *bundle.Bundle) error { return err } - root := BundleToTerraform(&b.Config) + var root *schema.Root + err = b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + root, err = BundleToTerraformWithDynValue(ctx, v) + return v, err + }) + if err != nil { + return err + } + f, err := os.Create(filepath.Join(dir, "bundle.tf.json")) if err != nil { return err diff --git a/libs/dyn/path.go b/libs/dyn/path.go index 34285de1..91893f92 100644 --- a/libs/dyn/path.go +++ b/libs/dyn/path.go @@ -10,6 +10,14 @@ type pathComponent struct { index int } +func (c pathComponent) Key() string { + return c.key +} + +func (c pathComponent) Index() int { + return c.index +} + func (c pathComponent) isKey() bool { return c.key != "" }