wrap `terraform` execution PoC

This commit is contained in:
Serge Smertin 2022-05-21 14:31:26 +02:00
parent b85e63684b
commit 3907dcdba9
6 changed files with 185 additions and 2 deletions

5
.gitignore vendored
View File

@ -20,4 +20,7 @@ dist/
coverage.txt
__pycache__
*.pyc
*.pyc
.terraform
.terraform.lock.hcl

6
go.mod
View File

@ -3,13 +3,17 @@ module github.com/databricks/bricks
go 1.16
require (
github.com/andybalholm/crlf v0.0.0-20171020200849-670099aa064f // indirect
github.com/databrickslabs/terraform-provider-databricks v0.5.7 // Apache 2.0
gopkg.in/ini.v1 v1.66.4 // Apache 2.0
github.com/ghodss/yaml v1.0.0 // MIT + NOTICE
github.com/hashicorp/go-version v1.5.0 // MPL 2.0
github.com/hashicorp/hc-install v0.3.2 // MPL 2.0
github.com/hashicorp/terraform-exec v0.16.1 // MPL 2.0
github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause license
github.com/mitchellh/go-homedir v1.1.0 // MIT
github.com/spf13/cobra v1.4.0 // Apache 2.0
github.com/stretchr/testify v1.7.1 // MIT
github.com/whilp/git-urls v1.0.0 // MIT
golang.org/x/mod v0.5.1 // BSD-3-Clause
gopkg.in/ini.v1 v1.66.4 // Apache 2.0
)

7
go.sum
View File

@ -84,6 +84,7 @@ github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ
github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
github.com/agext/levenshtein v1.2.2 h1:0S/Yg6LYmFJ5stwQeRp6EeOcCbj7xiqQSdNelsXvaqE=
github.com/agext/levenshtein v1.2.2/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
github.com/andybalholm/crlf v0.0.0-20171020200849-670099aa064f/go.mod h1:k8feO4+kXDxro6ErPXBRTJ/ro2mf0SsFG8s7doP9kJE=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU=
@ -206,6 +207,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@ -263,10 +266,14 @@ github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/go-version v1.4.0 h1:aAQzgqIrRKRa7w75CKpbBxYsmUoPjzVm1W59ca1L0J4=
github.com/hashicorp/go-version v1.4.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/go-version v1.5.0 h1:O293SZ2Eg+AAYijkVK3jR786Am1bhDEh2GHT0tIVE5E=
github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/hc-install v0.3.1 h1:VIjllE6KyAI1A244G8kTaHXy+TL5/XYzvrtFi8po/Yk=
github.com/hashicorp/hc-install v0.3.1/go.mod h1:3LCdWcCDS1gaHC9mhHCGbkYfoY6vdsKohGjugbZdZak=
github.com/hashicorp/hc-install v0.3.2 h1:oiQdJZvXmkNcRcEOOfM5n+VTsvNjWQeOjfAoO6dKSH8=
github.com/hashicorp/hc-install v0.3.2/go.mod h1:xMG6Tr8Fw1WFjlxH0A9v61cW15pFwgEGqEz0V4jisHs=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/hcl/v2 v2.11.1/go.mod h1:FwWsfWEjyV/CMj8s/gqAuiviY72rJ1/oayI9WftqcKg=

View File

@ -0,0 +1,41 @@
terraform {
required_providers {
databricks = {
source = "databrickslabs/databricks"
}
}
}
provider "databricks" {
}
# This file cannot be used for tests until `InsecureSkipVerify` is exposed though env var
# data "databricks_current_user" "me" {}
# data "databricks_spark_version" "latest" {}
# data "databricks_node_type" "smallest" {
# local_disk = true
# }
resource "databricks_notebook" "this" {
path = "/Users/me@example.com/Terraform"
language = "PYTHON"
content_base64 = base64encode(<<-EOT
# created from ${abspath(path.module)}
display(spark.range(10))
EOT
)
}
resource "databricks_job" "this" {
name = "Terraform Demo (me@example.com)"
new_cluster {
num_workers = 1
spark_version = "10.0.1"
node_type_id = "i3.xlarge"
}
notebook_task {
notebook_path = databricks_notebook.this.path
}
}

78
terraform/runner.go Normal file
View File

@ -0,0 +1,78 @@
/*
How to simplify terraform configuration for the project?
---
Solve the following adoption slowers:
- remove the need for `required_providers` block
- authenticate Databricks provider with the same DatabricksClient
- skip downloading and locking Databricks provider every time (few seconds)
- users won't have to copy-paste these into their configs:
```hcl
terraform {
required_providers {
databricks = {
source = "databrickslabs/databricks"
}
}
}
provider "databricks" {
}
```
Terraform Plugin SDK v2 is using similar techniques for testing providers. One may find
details in github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource/plugin.go. In short:
- init provider isntance
- start terraform plugin GRPC server
- "reattach" providers and specify the `tfexec.Reattach` options, which essentially
forward GRPC address to terraform subprocess.
- this can be done by either adding a source depenency on Databricks provider
or adding a special launch mode to it.
For now
---
Let's see how far we can get without GRPC magic.
*/
package terraform
import (
"context"
"github.com/hashicorp/go-version"
"github.com/hashicorp/hc-install/product"
"github.com/hashicorp/hc-install/releases"
"github.com/hashicorp/terraform-exec/tfexec"
)
// installs terraform to a temporary directory (for now)
func installTerraform(ctx context.Context) (string, error) {
// TODO: let configuration and/or environment variable specify
// terraform binary. Or detect if terraform is installed in the $PATH
installer := &releases.ExactVersion{
Product: product.Terraform,
Version: version.Must(version.NewVersion("1.1.0")),
}
return installer.Install(ctx)
}
func newTerraform(ctx context.Context, workDir string, env map[string]string) (*tfexec.Terraform, error) {
execPath, err := installTerraform(ctx)
if err != nil {
return nil, err
}
// TODO: figure out how to cleanup/skip `.terraform*` files and dirs, not to confuse users
// one of the options: take entire working directory with *.tf files and move them to tmpdir.
// make it optional, of course, otherwise debugging may become super hard.
tf, err := tfexec.NewTerraform(workDir, execPath)
if err != nil {
return nil, err
}
err = tf.SetEnv(env)
if err != nil {
return nil, err
}
return tf, err
}

50
terraform/runner_test.go Normal file
View File

@ -0,0 +1,50 @@
package terraform
import (
"context"
"path"
"testing"
"github.com/hashicorp/terraform-exec/tfexec"
"github.com/stretchr/testify/assert"
)
func TestSomething(t *testing.T) {
ctx := context.Background()
tf, err := newTerraform(ctx, "internal/test/simplest", map[string]string{
"DATABRICKS_HOST": "..",
"DATABRICKS_TOKEN": "..",
})
assert.NoError(t, err)
err = tf.Init(ctx)
assert.NoError(t, err)
planLoc := path.Join(t.TempDir(), "tfplan.zip")
hasChanges, err := tf.Plan(ctx, tfexec.Out(planLoc))
assert.True(t, hasChanges)
assert.NoError(t, err)
plan, err := tf.ShowPlanFile(ctx, planLoc)
assert.NoError(t, err)
assert.NotNil(t, plan)
found := false
for _, r := range plan.Config.RootModule.Resources {
// TODO: add validator to prevent non-Databricks resources in *.tf files, as
// we're not replacing terraform, we're wrapping it up for the average user.
if r.Type != "databricks_job" {
continue
}
found = true
// resource "databricks_job" "this"
assert.Equal(t, "this", r.Name)
// this is just a PoC to show how to retrieve DBR version from definitions.
// production code should perform rigorous nil checks...
nc := r.Expressions["new_cluster"]
firstBlock := nc.NestedBlocks[0]
ver := firstBlock["spark_version"].ConstantValue.(string)
assert.Equal(t, "10.0.1", ver)
}
assert.True(t, found)
}