Automatically install Terraform if needed (#141)

Users can opt out and use the system-installed version with the
following configuration:

```
bundle:
  terraform:
    exec_path: terraform
```

This will find the binary in $PATH and replace it with the found value.

If this is not set, the initialize phase will install Terraform in the
bundle's cache directory.

bundle/bundle.go

@@ -65,13 +65,26 @@ var cacheDirName = filepath.Join(".databricks", "bundle")
 
 // CacheDir returns directory to use for temporary files for this bundle.
 // Scoped to the bundle's environment.
-func (b *Bundle) CacheDir() (string, error) {
+func (b *Bundle) CacheDir(paths ...string) (string, error) {
 	if b.Config.Bundle.Environment == "" {
 		panic("environment not set")
 	}
 
+	// Fixed components of the result path.
+	parts := []string{
+		// Anchor at bundle root directory.
+		b.Config.Path,
+		// Static cache directory.
+		cacheDirName,
+		// Scope with environment name.
+		b.Config.Bundle.Environment,
+	}
+
+	// Append dynamic components of the result path.
+	parts = append(parts, paths...)
+
 	// Make directory if it doesn't exist yet.
-	dir := filepath.Join(b.Config.Path, cacheDirName, b.Config.Bundle.Environment)
+	dir := filepath.Join(parts...)
 	err := os.MkdirAll(dir, 0700)
 	if err != nil {
 		return "", err

bundle/config/bundle.go

@@ -1,5 +1,9 @@
 package config
 
+type Terraform struct {
+	ExecPath string `json:"exec_path"`
+}
+
 type Bundle struct {
 	Name string `json:"name,omitempty"`
 
@@ -13,4 +17,8 @@ type Bundle struct {
 
 	// Environment is set by the mutator that selects the environment.
 	Environment string `json:"environment,omitempty"`
+
+	// Terraform holds configuration related to Terraform.
+	// For example, where to find the binary, which version to use, etc.
+	Terraform *Terraform `json:"terraform,omitempty"`
 }

bundle/deploy/terraform/apply.go

@@ -3,7 +3,6 @@ package terraform
 import (
 	"context"
 	"fmt"
-	"os/exec"
 
 	"github.com/databricks/bricks/bundle"
 	"github.com/hashicorp/terraform-exec/tfexec"
@@ -16,22 +15,12 @@ func (w *apply) Name() string {
 }
 
 func (w *apply) Apply(ctx context.Context, b *bundle.Bundle) ([]bundle.Mutator, error) {
-	workingDir, err := Dir(b)
+	tf := b.Terraform
-	if err != nil {
+	if tf == nil {
-		return nil, err
+		return nil, fmt.Errorf("terraform not initialized")
 	}
 
-	execPath, err := exec.LookPath("terraform")
+	err := tf.Init(ctx, tfexec.Upgrade(true))
-	if err != nil {
-		return nil, err
-	}
-
-	tf, err := tfexec.NewTerraform(workingDir, execPath)
-	if err != nil {
-		return nil, err
-	}
-
-	err = tf.Init(ctx, tfexec.Upgrade(true))
 	if err != nil {
 		return nil, fmt.Errorf("terraform init: %w", err)
 	}

bundle/deploy/terraform/dir.go

@@ -1,25 +1,11 @@
 package terraform
 
 import (
-	"os"
-	"path/filepath"
-
 	"github.com/databricks/bricks/bundle"
 )
 
 // Dir returns the Terraform working directory for a given bundle.
 // The working directory is emphemeral and nested under the bundle's cache directory.
 func Dir(b *bundle.Bundle) (string, error) {
-	path, err := b.CacheDir()
+	return b.CacheDir("terraform")
-	if err != nil {
-		return "", err
-	}
-
-	nest := filepath.Join(path, "terraform")
-	err = os.MkdirAll(nest, 0700)
-	if err != nil {
-		return "", err
-	}
-
-	return nest, nil
 }

bundle/deploy/terraform/init.go

@@ -2,9 +2,17 @@ package terraform
 
 import (
 	"context"
+	"fmt"
+	"log"
+	"os"
 	"os/exec"
+	"path/filepath"
 
 	"github.com/databricks/bricks/bundle"
+	"github.com/databricks/bricks/bundle/config"
+	"github.com/hashicorp/go-version"
+	"github.com/hashicorp/hc-install/product"
+	"github.com/hashicorp/hc-install/releases"
 	"github.com/hashicorp/terraform-exec/tfexec"
 )
 
@@ -14,13 +22,64 @@ func (m *initialize) Name() string {
 	return "terraform.Initialize"
 }
 
+func (m *initialize) findExecPath(ctx context.Context, b *bundle.Bundle, tf *config.Terraform) (string, error) {
+	// If set, pass it through [exec.LookPath] to resolve its absolute path.
+	if tf.ExecPath != "" {
+		execPath, err := exec.LookPath(tf.ExecPath)
+		if err != nil {
+			return "", err
+		}
+		tf.ExecPath = execPath
+		log.Printf("[DEBUG] Using Terraform at %s", tf.ExecPath)
+		return tf.ExecPath, nil
+	}
+
+	binDir, err := b.CacheDir("bin")
+	if err != nil {
+		return "", err
+	}
+
+	// If the execPath already exists, return it.
+	execPath := filepath.Join(binDir, product.Terraform.BinaryName())
+	_, err = os.Stat(execPath)
+	if err != nil && !os.IsNotExist(err) {
+		return "", err
+	}
+	if err == nil {
+		tf.ExecPath = execPath
+		log.Printf("[DEBUG] Using Terraform at %s", tf.ExecPath)
+		return tf.ExecPath, nil
+	}
+
+	// Download Terraform to private bin directory.
+	installer := &releases.LatestVersion{
+		Product:     product.Terraform,
+		Constraints: version.MustConstraints(version.NewConstraint("<2.0")),
+		InstallDir:  binDir,
+	}
+	execPath, err = installer.Install(ctx)
+	if err != nil {
+		return "", fmt.Errorf("error downloading Terraform: %w", err)
+	}
+
+	tf.ExecPath = execPath
+	log.Printf("[DEBUG] Using Terraform at %s", tf.ExecPath)
+	return tf.ExecPath, nil
+}
+
 func (m *initialize) Apply(ctx context.Context, b *bundle.Bundle) ([]bundle.Mutator, error) {
-	workingDir, err := Dir(b)
+	tfConfig := b.Config.Bundle.Terraform
+	if tfConfig == nil {
+		tfConfig = &config.Terraform{}
+		b.Config.Bundle.Terraform = tfConfig
+	}
+
+	execPath, err := m.findExecPath(ctx, b, tfConfig)
 	if err != nil {
 		return nil, err
 	}
 
-	execPath, err := exec.LookPath("terraform")
+	workingDir, err := Dir(b)
 	if err != nil {
 		return nil, err
 	}

bundle/phases/initialize.go

@@ -4,6 +4,7 @@ import (
 	"github.com/databricks/bricks/bundle"
 	"github.com/databricks/bricks/bundle/config/interpolation"
 	"github.com/databricks/bricks/bundle/config/mutator"
+	"github.com/databricks/bricks/bundle/deploy/terraform"
 )
 
 // The initialize phase fills in defaults and connects to the workspace.
@@ -19,6 +20,7 @@ func Initialize() bundle.Mutator {
 				interpolation.IncludeLookupsInPath("bundle"),
 				interpolation.IncludeLookupsInPath("workspace"),
 			),
+			terraform.Initialize(),
 		},
 	)
 }