steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix lint errors for unused functions (#36) 

The functionality under terraform/ isn't used anywhere at the moment and
the test doesn't pass for me. It will be useful down the line so
commenting out instead of removing.

Confirmed that staticcheck passes when run with:
```
staticcheck -checks U1000 ./...
```
This commit is contained in:
Pieter Noordhuis 2022-09-07 14:26:31 +02:00 committed by GitHub
parent 67b2e4206f
commit 31a841ff33
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 179 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

301
terraform/runner.go
View File

@ -38,176 +38,157 @@ Let's see how far we can get without GRPC magic.
*/ */
package terraform package terraform
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"log"
"os"
"github.com/databricks/bricks/project"
"github.com/databricks/bricks/utilities"
"github.com/hashicorp/go-version"
"github.com/hashicorp/hc-install/product"
"github.com/hashicorp/hc-install/releases"
"github.com/hashicorp/terraform-exec/tfexec"
tfjson "github.com/hashicorp/terraform-json"
)
const DeploymentStateRemoteLocation = "dbfs:/FileStore/deployment-state" const DeploymentStateRemoteLocation = "dbfs:/FileStore/deployment-state"
type TerraformDeployer struct { // type TerraformDeployer struct {
WorkDir string // WorkDir string
CopyTfs bool // CopyTfs bool
tf *tfexec.Terraform // tf *tfexec.Terraform
} // }
func (d *TerraformDeployer) Init(ctx context.Context) error { // func (d *TerraformDeployer) Init(ctx context.Context) error {
if d.CopyTfs { // if d.CopyTfs {
panic("copying tf configuration files to a temporary dir not yet implemented") // panic("copying tf configuration files to a temporary dir not yet implemented")
} // }
// TODO: most likely merge the methods // // TODO: most likely merge the methods
exec, err := newTerraform(ctx, d.WorkDir, map[string]string{}) // exec, err := newTerraform(ctx, d.WorkDir, map[string]string{})
if err != nil { // if err != nil {
return err // return err
} // }
d.tf = exec // d.tf = exec
return nil // return nil
} // }
// returns location of terraform state on DBFS based on project's deployment isolation level. // // returns location of terraform state on DBFS based on project's deployment isolation level.
func (d *TerraformDeployer) remoteTfstateLoc() string { // func (d *TerraformDeployer) remoteTfstateLoc() string {
prefix := project.Current.DeploymentIsolationPrefix() // prefix := project.Current.DeploymentIsolationPrefix()
return fmt.Sprintf("%s/%s/terraform.tfstate", DeploymentStateRemoteLocation, prefix) // return fmt.Sprintf("%s/%s/terraform.tfstate", DeploymentStateRemoteLocation, prefix)
} // }
// returns structured representation of terraform state on DBFS. // // returns structured representation of terraform state on DBFS.
func (d *TerraformDeployer) remoteState(ctx context.Context) (*tfjson.State, int, error) { // func (d *TerraformDeployer) remoteState(ctx context.Context) (*tfjson.State, int, error) {
raw, err := utilities.ReadDbfsFile(ctx, // raw, err := utilities.ReadDbfsFile(ctx,
project.Current.WorkspacesClient(), // project.Current.WorkspacesClient(),
d.remoteTfstateLoc(), // d.remoteTfstateLoc(),
) // )
if err != nil { // if err != nil {
return nil, 0, err // return nil, 0, err
} // }
return d.tfstateFromReader(bytes.NewBuffer(raw)) // return d.tfstateFromReader(bytes.NewBuffer(raw))
} // }
// opens file handle for local-backend terraform state, that has to be closed in the calling // // opens file handle for local-backend terraform state, that has to be closed in the calling
// methods. this file alone is not the authoritative state of deployment and has to properly // // methods. this file alone is not the authoritative state of deployment and has to properly
// be synced with remote counterpart. // // be synced with remote counterpart.
func (d *TerraformDeployer) openLocalState() (*os.File, error) { // func (d *TerraformDeployer) openLocalState() (*os.File, error) {
return os.Open(fmt.Sprintf("%s/terraform.tfstate", d.WorkDir)) // return os.Open(fmt.Sprintf("%s/terraform.tfstate", d.WorkDir))
} // }
// returns structured representation of terraform state on local machine. as part of // // returns structured representation of terraform state on local machine. as part of
// the optimistic concurrency control, please make sure to always compare the serial // // the optimistic concurrency control, please make sure to always compare the serial
// number of local and remote states before proceeding with deployment. // // number of local and remote states before proceeding with deployment.
func (d *TerraformDeployer) localState() (*tfjson.State, int, error) { // func (d *TerraformDeployer) localState() (*tfjson.State, int, error) {
local, err := d.openLocalState() // local, err := d.openLocalState()
if err != nil { // if err != nil {
return nil, 0, err // return nil, 0, err
} // }
defer local.Close() // defer local.Close()
return d.tfstateFromReader(local) // return d.tfstateFromReader(local)
} // }
// converts input stream into structured representation of terraform state and deployment // // converts input stream into structured representation of terraform state and deployment
// serial number, that helps controlling versioning and synchronisation via optimistic locking. // // serial number, that helps controlling versioning and synchronisation via optimistic locking.
func (d *TerraformDeployer) tfstateFromReader(reader io.Reader) (*tfjson.State, int, error) { // func (d *TerraformDeployer) tfstateFromReader(reader io.Reader) (*tfjson.State, int, error) {
var state tfjson.State // var state tfjson.State
state.UseJSONNumber(true) // state.UseJSONNumber(true)
decoder := json.NewDecoder(reader) // decoder := json.NewDecoder(reader)
decoder.UseNumber() // decoder.UseNumber()
err := decoder.Decode(&state) // err := decoder.Decode(&state)
if err != nil { // if err != nil {
return nil, 0, err // return nil, 0, err
} // }
err = state.Validate() // err = state.Validate()
if err != nil { // if err != nil {
return nil, 0, err // return nil, 0, err
} // }
var serialWrapper struct { // var serialWrapper struct {
Serial int `json:"serial,omitempty"` // Serial int `json:"serial,omitempty"`
} // }
// TODO: use byte buffer if this decoder fails on double reading // // TODO: use byte buffer if this decoder fails on double reading
err = decoder.Decode(&serialWrapper) // err = decoder.Decode(&serialWrapper)
if err != nil { // if err != nil {
return nil, 0, err // return nil, 0, err
} // }
return &state, serialWrapper.Serial, nil // return &state, serialWrapper.Serial, nil
} // }
// uploads terraform state from local directory to designated DBFS location. // // uploads terraform state from local directory to designated DBFS location.
func (d *TerraformDeployer) uploadTfstate(ctx context.Context) error { // func (d *TerraformDeployer) uploadTfstate(ctx context.Context) error {
local, err := d.openLocalState() // local, err := d.openLocalState()
if err != nil { // if err != nil {
return err // return err
} // }
defer local.Close() // defer local.Close()
raw, err := io.ReadAll(local) // raw, err := io.ReadAll(local)
if err != nil { // if err != nil {
return err // return err
} // }
// TODO: make sure that deployment locks are implemented // // TODO: make sure that deployment locks are implemented
return utilities.CreateDbfsFile(ctx, // return utilities.CreateDbfsFile(ctx,
project.Current.WorkspacesClient(), // project.Current.WorkspacesClient(),
d.remoteTfstateLoc(), // d.remoteTfstateLoc(),
raw, // raw,
true, // true,
) // )
} // }
// downloads terraform state from DBFS to local working directory. // // downloads terraform state from DBFS to local working directory.
func (d *TerraformDeployer) downloadTfstate(ctx context.Context) error { // func (d *TerraformDeployer) downloadTfstate(ctx context.Context) error {
remote, serialDeployed, err := d.remoteState(ctx) // remote, serialDeployed, err := d.remoteState(ctx)
if err != nil { // if err != nil {
return err // return err
} // }
log.Printf("[DEBUG] remote serial is %d", serialDeployed) // log.Printf("[DEBUG] remote serial is %d", serialDeployed)
local, err := d.openLocalState() // local, err := d.openLocalState()
if err != nil { // if err != nil {
return err // return err
} // }
defer local.Close() // defer local.Close()
raw, err := json.Marshal(remote) // raw, err := json.Marshal(remote)
if err != nil { // if err != nil {
return err // return err
} // }
_, err = io.Copy(local, bytes.NewBuffer(raw)) // _, err = io.Copy(local, bytes.NewBuffer(raw))
return err // return err
} // }
// installs terraform to a temporary directory (for now) // // installs terraform to a temporary directory (for now)
func installTerraform(ctx context.Context) (string, error) { // func installTerraform(ctx context.Context) (string, error) {
// TODO: let configuration and/or environment variable specify // // TODO: let configuration and/or environment variable specify
// terraform binary. Or detect if terraform is installed in the $PATH // // terraform binary. Or detect if terraform is installed in the $PATH
installer := &releases.ExactVersion{ // installer := &releases.ExactVersion{
Product: product.Terraform, // Product: product.Terraform,
Version: version.Must(version.NewVersion("1.1.0")), // Version: version.Must(version.NewVersion("1.1.0")),
} // }
return installer.Install(ctx) // return installer.Install(ctx)
} // }
func newTerraform(ctx context.Context, workDir string, env map[string]string) (*tfexec.Terraform, error) { // func newTerraform(ctx context.Context, workDir string, env map[string]string) (*tfexec.Terraform, error) {
execPath, err := installTerraform(ctx) // execPath, err := installTerraform(ctx)
if err != nil { // if err != nil {
return nil, err // return nil, err
} // }
// TODO: figure out how to cleanup/skip `.terraform*` files and dirs, not to confuse users // // TODO: figure out how to cleanup/skip `.terraform*` files and dirs, not to confuse users
// one of the options: take entire working directory with *.tf files and move them to tmpdir. // // one of the options: take entire working directory with *.tf files and move them to tmpdir.
// make it optional, of course, otherwise debugging may become super hard. // // make it optional, of course, otherwise debugging may become super hard.
tf, err := tfexec.NewTerraform(workDir, execPath) // tf, err := tfexec.NewTerraform(workDir, execPath)
if err != nil { // if err != nil {
return nil, err // return nil, err
} // }
err = tf.SetEnv(env) // err = tf.SetEnv(env)
if err != nil { // if err != nil {
return nil, err // return nil, err
} // }
return tf, err // return tf, err
} // }

85
terraform/runner_test.go
View File

@ -1,53 +1,44 @@
package terraform package terraform
import ( // func TestSomething(t *testing.T) {
"context" // ctx := context.Background()
"path" // tf, err := newTerraform(ctx, "testdata/simplest", map[string]string{
"testing" // "DATABRICKS_HOST": "..",
// "DATABRICKS_TOKEN": "..",
// })
// assert.NoError(t, err)
"github.com/hashicorp/terraform-exec/tfexec" // err = tf.Init(ctx)
"github.com/stretchr/testify/assert" // assert.NoError(t, err)
)
func TestSomething(t *testing.T) { // planLoc := path.Join(t.TempDir(), "tfplan.zip")
ctx := context.Background() // hasChanges, err := tf.Plan(ctx, tfexec.Out(planLoc))
tf, err := newTerraform(ctx, "testdata/simplest", map[string]string{ // assert.True(t, hasChanges)
"DATABRICKS_HOST": "..", // assert.NoError(t, err)
"DATABRICKS_TOKEN": "..",
})
assert.NoError(t, err)
err = tf.Init(ctx) // plan, err := tf.ShowPlanFile(ctx, planLoc)
assert.NoError(t, err) // assert.NoError(t, err)
// assert.NotNil(t, plan)
planLoc := path.Join(t.TempDir(), "tfplan.zip") // found := false
hasChanges, err := tf.Plan(ctx, tfexec.Out(planLoc)) // for _, r := range plan.Config.RootModule.Resources {
assert.True(t, hasChanges) // // TODO: add validator to prevent non-Databricks resources in *.tf files, as
assert.NoError(t, err) // // we're not replacing terraform, we're wrapping it up for the average user.
// if r.Type != "databricks_job" {
plan, err := tf.ShowPlanFile(ctx, planLoc) // continue
assert.NoError(t, err) // }
assert.NotNil(t, plan) // // TODO: validate that libraries on jobs defined in *.tf and libraries
// // in `install_requires` defined in setup.py are the same. Exist with
found := false // // the explanatory error otherwise.
for _, r := range plan.Config.RootModule.Resources { // found = true
// TODO: add validator to prevent non-Databricks resources in *.tf files, as // // resource "databricks_job" "this"
// we're not replacing terraform, we're wrapping it up for the average user. // assert.Equal(t, "this", r.Name)
if r.Type != "databricks_job" { // // this is just a PoC to show how to retrieve DBR version from definitions.
continue // // production code should perform rigorous nil checks...
} // nc := r.Expressions["new_cluster"]
// TODO: validate that libraries on jobs defined in *.tf and libraries // firstBlock := nc.NestedBlocks[0]
// in `install_requires` defined in setup.py are the same. Exist with // ver := firstBlock["spark_version"].ConstantValue.(string)
// the explanatory error otherwise. // assert.Equal(t, "10.0.1", ver)
found = true // }
// resource "databricks_job" "this" // assert.True(t, found)
assert.Equal(t, "this", r.Name) // }
// this is just a PoC to show how to retrieve DBR version from definitions.
// production code should perform rigorous nil checks...
nc := r.Expressions["new_cluster"]
firstBlock := nc.NestedBlocks[0]
ver := firstBlock["spark_version"].ConstantValue.(string)
assert.Equal(t, "10.0.1", ver)
}
assert.True(t, found)
}