databricks-cli/terraform/runner.go

/*
How to simplify terraform configuration for the project?
---

Solve the following adoption slowers:

- remove the need for `required_providers` block
- authenticate Databricks provider with the same DatabricksClient
- skip downloading and locking Databricks provider every time (few seconds)
- users won't have to copy-paste these into their configs:

```hcl
terraform {
  required_providers {
    databricks = {
      source  = "databrickslabs/databricks"
    }
  }
}

provider "databricks" {
}
```

Terraform Plugin SDK v2 is using similar techniques for testing providers. One may find
details in github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource/plugin.go. In short:

- init provider isntance
- start terraform plugin GRPC server
- "reattach" providers and specify the `tfexec.Reattach` options, which essentially
  forward GRPC address to terraform subprocess.
- this can be done by either adding a source depenency on Databricks provider
  or adding a special launch mode to it.

For now
---
Let's see how far we can get without GRPC magic.
*/
package terraform

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"

	"github.com/databricks/bricks/project"
	"github.com/databrickslabs/terraform-provider-databricks/storage"
	"github.com/hashicorp/go-version"
	"github.com/hashicorp/hc-install/product"
	"github.com/hashicorp/hc-install/releases"
	"github.com/hashicorp/terraform-exec/tfexec"

	tfjson "github.com/hashicorp/terraform-json"
)

const DeploymentStateRemoteLocation = "dbfs:/FileStore/deployment-state"

type TerraformDeployer struct {
	WorkDir string
	CopyTfs bool

	tf *tfexec.Terraform
}

func (d *TerraformDeployer) Init(ctx context.Context) error {
	if d.CopyTfs {
		panic("copying tf configuration files to a temporary dir not yet implemented")
	}
	// TODO: most likely merge the methods
	exec, err := newTerraform(ctx, d.WorkDir, map[string]string{})
	if err != nil {
		return err
	}
	d.tf = exec
	return nil
}

// returns location of terraform state on DBFS based on project's deployment isolation level.
func (d *TerraformDeployer) remoteTfstateLoc() string {
	prefix := project.Current.DeploymentIsolationPrefix()
	return fmt.Sprintf("%s/%s/terraform.tfstate", DeploymentStateRemoteLocation, prefix)
}

// returns structured representation of terraform state on DBFS.
func (d *TerraformDeployer) remoteState(ctx context.Context) (*tfjson.State, int, error) {
	dbfs := storage.NewDbfsAPI(ctx, project.Current.Client())
	raw, err := dbfs.Read(d.remoteTfstateLoc())
	if err != nil {
		return nil, 0, err
	}
	return d.tfstateFromReader(bytes.NewBuffer(raw))
}

// opens file handle for local-backend terraform state, that has to be closed in the calling
// methods. this file alone is not the authoritative state of deployment and has to properly
// be synced with remote counterpart.
func (d *TerraformDeployer) openLocalState() (*os.File, error) {
	return os.Open(fmt.Sprintf("%s/terraform.tfstate", d.WorkDir))
}

// returns structured representation of terraform state on local machine. as part of 
// the optimistic concurrency control, please make sure to always compare the serial 
// number of local and remote states before proceeding with deployment.
func (d *TerraformDeployer) localState() (*tfjson.State, int, error) {
	local, err := d.openLocalState()
	if err != nil {
		return nil, 0, err
	}
	defer local.Close()
	return d.tfstateFromReader(local)
}

// converts input stream into structured representation of terraform state and deployment 
// serial number, that helps controlling versioning and synchronisation via optimistic locking.
func (d *TerraformDeployer) tfstateFromReader(reader io.Reader) (*tfjson.State, int, error) {
	var state tfjson.State
	state.UseJSONNumber(true)
	decoder := json.NewDecoder(reader)
	decoder.UseNumber()
	err := decoder.Decode(&state)
	if err != nil {
		return nil, 0, err
	}
	err = state.Validate()
	if err != nil {
		return nil, 0, err
	}
	var serialWrapper struct {
		Serial int `json:"serial,omitempty"`
	}
	// TODO: use byte buffer if this decoder fails on double reading
	err = decoder.Decode(&serialWrapper)
	if err != nil {
		return nil, 0, err
	}
	return &state, serialWrapper.Serial, nil
}

// uploads terraform state from local directory to designated DBFS location.
func (d *TerraformDeployer) uploadTfstate(ctx context.Context) error {
	dbfs := storage.NewDbfsAPI(ctx, project.Current.Client())
	local, err := d.openLocalState()
	if err != nil {
		return err
	}
	defer local.Close()
	raw, err := io.ReadAll(local)
	if err != nil {
		return err
	}
	// TODO: make sure that deployment locks are implemented
	return dbfs.Create(d.remoteTfstateLoc(), raw, true)
}

// downloads terraform state from DBFS to local working directory.
func (d *TerraformDeployer) downloadTfstate(ctx context.Context) error {
	remote, serialDeployed, err := d.remoteState(ctx)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] remote serial is %d", serialDeployed)
	local, err := d.openLocalState()
	if err != nil {
		return err
	}
	defer local.Close()
	raw, err := json.Marshal(remote)
	if err != nil {
		return err
	}
	_, err = io.Copy(local, bytes.NewBuffer(raw))
	return err
}

// installs terraform to a temporary directory (for now)
func installTerraform(ctx context.Context) (string, error) {
	// TODO: let configuration and/or environment variable specify
	// terraform binary. Or detect if terraform is installed in the $PATH
	installer := &releases.ExactVersion{
		Product: product.Terraform,
		Version: version.Must(version.NewVersion("1.1.0")),
	}
	return installer.Install(ctx)
}

func newTerraform(ctx context.Context, workDir string, env map[string]string) (*tfexec.Terraform, error) {
	execPath, err := installTerraform(ctx)
	if err != nil {
		return nil, err
	}
	// TODO: figure out how to cleanup/skip `.terraform*` files and dirs, not to confuse users
	// one of the options: take entire working directory with *.tf files and move them to tmpdir.
	// make it optional, of course, otherwise debugging may become super hard.
	tf, err := tfexec.NewTerraform(workDir, execPath)
	if err != nil {
		return nil, err
	}
	err = tf.SetEnv(env)
	if err != nil {
		return nil, err
	}
	return tf, err
}