wrap `terraform` execution PoC

.gitignore

@@ -21,3 +21,6 @@ coverage.txt
 
 __pycache__
 *.pyc
+
+.terraform
+.terraform.lock.hcl

go.mod

@@ -3,13 +3,17 @@ module github.com/databricks/bricks
 go 1.16
 
 require (
+	github.com/andybalholm/crlf v0.0.0-20171020200849-670099aa064f // indirect
 	github.com/databrickslabs/terraform-provider-databricks v0.5.7 // Apache 2.0
-	gopkg.in/ini.v1 v1.66.4 // Apache 2.0
 	github.com/ghodss/yaml v1.0.0 // MIT + NOTICE
+	github.com/hashicorp/go-version v1.5.0 // MPL 2.0
+	github.com/hashicorp/hc-install v0.3.2 // MPL 2.0
+	github.com/hashicorp/terraform-exec v0.16.1 // MPL 2.0
 	github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause license
 	github.com/mitchellh/go-homedir v1.1.0 // MIT
 	github.com/spf13/cobra v1.4.0 // Apache 2.0
 	github.com/stretchr/testify v1.7.1 // MIT
 	github.com/whilp/git-urls v1.0.0 // MIT
 	golang.org/x/mod v0.5.1 // BSD-3-Clause
+	gopkg.in/ini.v1 v1.66.4 // Apache 2.0
 )

go.sum

@@ -84,6 +84,7 @@ github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ
 github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
 github.com/agext/levenshtein v1.2.2 h1:0S/Yg6LYmFJ5stwQeRp6EeOcCbj7xiqQSdNelsXvaqE=
 github.com/agext/levenshtein v1.2.2/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
+github.com/andybalholm/crlf v0.0.0-20171020200849-670099aa064f/go.mod h1:k8feO4+kXDxro6ErPXBRTJ/ro2mf0SsFG8s7doP9kJE=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU=
@@ -206,6 +207,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@@ -263,10 +266,14 @@ github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
 github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.4.0 h1:aAQzgqIrRKRa7w75CKpbBxYsmUoPjzVm1W59ca1L0J4=
 github.com/hashicorp/go-version v1.4.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.5.0 h1:O293SZ2Eg+AAYijkVK3jR786Am1bhDEh2GHT0tIVE5E=
+github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hc-install v0.3.1 h1:VIjllE6KyAI1A244G8kTaHXy+TL5/XYzvrtFi8po/Yk=
 github.com/hashicorp/hc-install v0.3.1/go.mod h1:3LCdWcCDS1gaHC9mhHCGbkYfoY6vdsKohGjugbZdZak=
+github.com/hashicorp/hc-install v0.3.2 h1:oiQdJZvXmkNcRcEOOfM5n+VTsvNjWQeOjfAoO6dKSH8=
+github.com/hashicorp/hc-install v0.3.2/go.mod h1:xMG6Tr8Fw1WFjlxH0A9v61cW15pFwgEGqEz0V4jisHs=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/hcl/v2 v2.11.1/go.mod h1:FwWsfWEjyV/CMj8s/gqAuiviY72rJ1/oayI9WftqcKg=

terraform/internal/test/simplest/demo.tf

@@ -0,0 +1,41 @@
+terraform {
+  required_providers {
+    databricks = {
+      source  = "databrickslabs/databricks"
+    }
+  }
+}
+
+provider "databricks" {
+}
+
+# This file cannot be used for tests until `InsecureSkipVerify` is exposed though env var
+# data "databricks_current_user" "me" {}
+# data "databricks_spark_version" "latest" {}
+# data "databricks_node_type" "smallest" {
+#   local_disk = true
+# }
+
+resource "databricks_notebook" "this" {
+  path     = "/Users/me@example.com/Terraform"
+  language = "PYTHON"
+  content_base64 = base64encode(<<-EOT
+    # created from ${abspath(path.module)}
+    display(spark.range(10))
+    EOT
+  )
+}
+
+resource "databricks_job" "this" {
+  name = "Terraform Demo (me@example.com)"
+
+  new_cluster {
+    num_workers   = 1
+    spark_version = "10.0.1"
+    node_type_id  = "i3.xlarge"
+  }
+
+  notebook_task {
+    notebook_path = databricks_notebook.this.path
+  }
+}

terraform/runner.go

@@ -0,0 +1,78 @@
+/*
+How to simplify terraform configuration for the project?
+---
+
+Solve the following adoption slowers:
+
+- remove the need for `required_providers` block
+- authenticate Databricks provider with the same DatabricksClient
+- skip downloading and locking Databricks provider every time (few seconds)
+- users won't have to copy-paste these into their configs:
+
+```hcl
+terraform {
+  required_providers {
+    databricks = {
+      source  = "databrickslabs/databricks"
+    }
+  }
+}
+
+provider "databricks" {
+}
+```
+
+Terraform Plugin SDK v2 is using similar techniques for testing providers. One may find
+details in github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource/plugin.go. In short:
+
+- init provider isntance
+- start terraform plugin GRPC server
+- "reattach" providers and specify the `tfexec.Reattach` options, which essentially
+  forward GRPC address to terraform subprocess.
+- this can be done by either adding a source depenency on Databricks provider
+  or adding a special launch mode to it.
+
+For now
+---
+Let's see how far we can get without GRPC magic.
+*/
+package terraform
+
+import (
+	"context"
+
+	"github.com/hashicorp/go-version"
+	"github.com/hashicorp/hc-install/product"
+	"github.com/hashicorp/hc-install/releases"
+	"github.com/hashicorp/terraform-exec/tfexec"
+)
+
+// installs terraform to a temporary directory (for now)
+func installTerraform(ctx context.Context) (string, error) {
+	// TODO: let configuration and/or environment variable specify
+	// terraform binary. Or detect if terraform is installed in the $PATH
+	installer := &releases.ExactVersion{
+		Product: product.Terraform,
+		Version: version.Must(version.NewVersion("1.1.0")),
+	}
+	return installer.Install(ctx)
+}
+
+func newTerraform(ctx context.Context, workDir string, env map[string]string) (*tfexec.Terraform, error) {
+	execPath, err := installTerraform(ctx)
+	if err != nil {
+		return nil, err
+	}
+	// TODO: figure out how to cleanup/skip `.terraform*` files and dirs, not to confuse users
+	// one of the options: take entire working directory with *.tf files and move them to tmpdir.
+	// make it optional, of course, otherwise debugging may become super hard.
+	tf, err := tfexec.NewTerraform(workDir, execPath)
+	if err != nil {
+		return nil, err
+	}
+	err = tf.SetEnv(env)
+	if err != nil {
+		return nil, err
+	}
+	return tf, err
+}

terraform/runner_test.go

@@ -0,0 +1,50 @@
+package terraform
+
+import (
+	"context"
+	"path"
+	"testing"
+
+	"github.com/hashicorp/terraform-exec/tfexec"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSomething(t *testing.T) {
+	ctx := context.Background()
+	tf, err := newTerraform(ctx, "internal/test/simplest", map[string]string{
+		"DATABRICKS_HOST":  "..",
+		"DATABRICKS_TOKEN": "..",
+	})
+	assert.NoError(t, err)
+
+	err = tf.Init(ctx)
+	assert.NoError(t, err)
+
+	planLoc := path.Join(t.TempDir(), "tfplan.zip")
+	hasChanges, err := tf.Plan(ctx, tfexec.Out(planLoc))
+	assert.True(t, hasChanges)
+	assert.NoError(t, err)
+
+	plan, err := tf.ShowPlanFile(ctx, planLoc)
+	assert.NoError(t, err)
+	assert.NotNil(t, plan)
+
+	found := false
+	for _, r := range plan.Config.RootModule.Resources {
+		// TODO: add validator to prevent non-Databricks resources in *.tf files, as
+		// we're not replacing terraform, we're wrapping it up for the average user.
+		if r.Type != "databricks_job" {
+			continue
+		}
+		found = true
+		// resource "databricks_job" "this"
+		assert.Equal(t, "this", r.Name)
+		// this is just a PoC to show how to retrieve DBR version from definitions.
+		// production code should perform rigorous nil checks...
+		nc := r.Expressions["new_cluster"]
+		firstBlock := nc.NestedBlocks[0]
+		ver := firstBlock["spark_version"].ConstantValue.(string)
+		assert.Equal(t, "10.0.1", ver)
+	}
+	assert.True(t, found)
+}