From 3907dcdba986cf7c40e62b5829e6ab60d3e88ad5 Mon Sep 17 00:00:00 2001 From: Serge Smertin Date: Sat, 21 May 2022 14:31:26 +0200 Subject: [PATCH] wrap `terraform` execution PoC --- .gitignore | 5 +- go.mod | 6 +- go.sum | 7 +++ terraform/internal/test/simplest/demo.tf | 41 +++++++++++++ terraform/runner.go | 78 ++++++++++++++++++++++++ terraform/runner_test.go | 50 +++++++++++++++ 6 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 terraform/internal/test/simplest/demo.tf create mode 100644 terraform/runner.go create mode 100644 terraform/runner_test.go diff --git a/.gitignore b/.gitignore index 8bbb04e0..176f02c0 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,7 @@ dist/ coverage.txt __pycache__ -*.pyc \ No newline at end of file +*.pyc + +.terraform +.terraform.lock.hcl \ No newline at end of file diff --git a/go.mod b/go.mod index f725137e..d2ccf59b 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,17 @@ module github.com/databricks/bricks go 1.16 require ( + github.com/andybalholm/crlf v0.0.0-20171020200849-670099aa064f // indirect github.com/databrickslabs/terraform-provider-databricks v0.5.7 // Apache 2.0 - gopkg.in/ini.v1 v1.66.4 // Apache 2.0 github.com/ghodss/yaml v1.0.0 // MIT + NOTICE + github.com/hashicorp/go-version v1.5.0 // MPL 2.0 + github.com/hashicorp/hc-install v0.3.2 // MPL 2.0 + github.com/hashicorp/terraform-exec v0.16.1 // MPL 2.0 github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause license github.com/mitchellh/go-homedir v1.1.0 // MIT github.com/spf13/cobra v1.4.0 // Apache 2.0 github.com/stretchr/testify v1.7.1 // MIT github.com/whilp/git-urls v1.0.0 // MIT golang.org/x/mod v0.5.1 // BSD-3-Clause + gopkg.in/ini.v1 v1.66.4 // Apache 2.0 ) diff --git a/go.sum b/go.sum index 3b78e1dd..2bf34558 100644 --- a/go.sum +++ b/go.sum @@ -84,6 +84,7 @@ github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/agext/levenshtein v1.2.2 h1:0S/Yg6LYmFJ5stwQeRp6EeOcCbj7xiqQSdNelsXvaqE= github.com/agext/levenshtein v1.2.2/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/andybalholm/crlf v0.0.0-20171020200849-670099aa064f/go.mod h1:k8feO4+kXDxro6ErPXBRTJ/ro2mf0SsFG8s7doP9kJE= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU= @@ -206,6 +207,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -263,10 +266,14 @@ github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.4.0 h1:aAQzgqIrRKRa7w75CKpbBxYsmUoPjzVm1W59ca1L0J4= github.com/hashicorp/go-version v1.4.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.5.0 h1:O293SZ2Eg+AAYijkVK3jR786Am1bhDEh2GHT0tIVE5E= +github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hc-install v0.3.1 h1:VIjllE6KyAI1A244G8kTaHXy+TL5/XYzvrtFi8po/Yk= github.com/hashicorp/hc-install v0.3.1/go.mod h1:3LCdWcCDS1gaHC9mhHCGbkYfoY6vdsKohGjugbZdZak= +github.com/hashicorp/hc-install v0.3.2 h1:oiQdJZvXmkNcRcEOOfM5n+VTsvNjWQeOjfAoO6dKSH8= +github.com/hashicorp/hc-install v0.3.2/go.mod h1:xMG6Tr8Fw1WFjlxH0A9v61cW15pFwgEGqEz0V4jisHs= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl/v2 v2.11.1/go.mod h1:FwWsfWEjyV/CMj8s/gqAuiviY72rJ1/oayI9WftqcKg= diff --git a/terraform/internal/test/simplest/demo.tf b/terraform/internal/test/simplest/demo.tf new file mode 100644 index 00000000..f4a8db57 --- /dev/null +++ b/terraform/internal/test/simplest/demo.tf @@ -0,0 +1,41 @@ +terraform { + required_providers { + databricks = { + source = "databrickslabs/databricks" + } + } +} + +provider "databricks" { +} + +# This file cannot be used for tests until `InsecureSkipVerify` is exposed though env var +# data "databricks_current_user" "me" {} +# data "databricks_spark_version" "latest" {} +# data "databricks_node_type" "smallest" { +# local_disk = true +# } + +resource "databricks_notebook" "this" { + path = "/Users/me@example.com/Terraform" + language = "PYTHON" + content_base64 = base64encode(<<-EOT + # created from ${abspath(path.module)} + display(spark.range(10)) + EOT + ) +} + +resource "databricks_job" "this" { + name = "Terraform Demo (me@example.com)" + + new_cluster { + num_workers = 1 + spark_version = "10.0.1" + node_type_id = "i3.xlarge" + } + + notebook_task { + notebook_path = databricks_notebook.this.path + } +} \ No newline at end of file diff --git a/terraform/runner.go b/terraform/runner.go new file mode 100644 index 00000000..452dedc4 --- /dev/null +++ b/terraform/runner.go @@ -0,0 +1,78 @@ +/* +How to simplify terraform configuration for the project? +--- + +Solve the following adoption slowers: + +- remove the need for `required_providers` block +- authenticate Databricks provider with the same DatabricksClient +- skip downloading and locking Databricks provider every time (few seconds) +- users won't have to copy-paste these into their configs: + +```hcl +terraform { + required_providers { + databricks = { + source = "databrickslabs/databricks" + } + } +} + +provider "databricks" { +} +``` + +Terraform Plugin SDK v2 is using similar techniques for testing providers. One may find +details in github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource/plugin.go. In short: + +- init provider isntance +- start terraform plugin GRPC server +- "reattach" providers and specify the `tfexec.Reattach` options, which essentially + forward GRPC address to terraform subprocess. +- this can be done by either adding a source depenency on Databricks provider + or adding a special launch mode to it. + +For now +--- +Let's see how far we can get without GRPC magic. +*/ +package terraform + +import ( + "context" + + "github.com/hashicorp/go-version" + "github.com/hashicorp/hc-install/product" + "github.com/hashicorp/hc-install/releases" + "github.com/hashicorp/terraform-exec/tfexec" +) + +// installs terraform to a temporary directory (for now) +func installTerraform(ctx context.Context) (string, error) { + // TODO: let configuration and/or environment variable specify + // terraform binary. Or detect if terraform is installed in the $PATH + installer := &releases.ExactVersion{ + Product: product.Terraform, + Version: version.Must(version.NewVersion("1.1.0")), + } + return installer.Install(ctx) +} + +func newTerraform(ctx context.Context, workDir string, env map[string]string) (*tfexec.Terraform, error) { + execPath, err := installTerraform(ctx) + if err != nil { + return nil, err + } + // TODO: figure out how to cleanup/skip `.terraform*` files and dirs, not to confuse users + // one of the options: take entire working directory with *.tf files and move them to tmpdir. + // make it optional, of course, otherwise debugging may become super hard. + tf, err := tfexec.NewTerraform(workDir, execPath) + if err != nil { + return nil, err + } + err = tf.SetEnv(env) + if err != nil { + return nil, err + } + return tf, err +} diff --git a/terraform/runner_test.go b/terraform/runner_test.go new file mode 100644 index 00000000..abd93946 --- /dev/null +++ b/terraform/runner_test.go @@ -0,0 +1,50 @@ +package terraform + +import ( + "context" + "path" + "testing" + + "github.com/hashicorp/terraform-exec/tfexec" + "github.com/stretchr/testify/assert" +) + +func TestSomething(t *testing.T) { + ctx := context.Background() + tf, err := newTerraform(ctx, "internal/test/simplest", map[string]string{ + "DATABRICKS_HOST": "..", + "DATABRICKS_TOKEN": "..", + }) + assert.NoError(t, err) + + err = tf.Init(ctx) + assert.NoError(t, err) + + planLoc := path.Join(t.TempDir(), "tfplan.zip") + hasChanges, err := tf.Plan(ctx, tfexec.Out(planLoc)) + assert.True(t, hasChanges) + assert.NoError(t, err) + + plan, err := tf.ShowPlanFile(ctx, planLoc) + assert.NoError(t, err) + assert.NotNil(t, plan) + + found := false + for _, r := range plan.Config.RootModule.Resources { + // TODO: add validator to prevent non-Databricks resources in *.tf files, as + // we're not replacing terraform, we're wrapping it up for the average user. + if r.Type != "databricks_job" { + continue + } + found = true + // resource "databricks_job" "this" + assert.Equal(t, "this", r.Name) + // this is just a PoC to show how to retrieve DBR version from definitions. + // production code should perform rigorous nil checks... + nc := r.Expressions["new_cluster"] + firstBlock := nc.NestedBlocks[0] + ver := firstBlock["spark_version"].ConstantValue.(string) + assert.Equal(t, "10.0.1", ver) + } + assert.True(t, found) +}