Add `bundle debug terraform` command (#1294)

- Add `bundle debug terraform` command. It prints versions of the
Terraform and the Databricks Terraform provider. In the text mode it
also explains how to setup the CLI in environments with restricted
internet access.
- Use `DATABRICKS_TF_EXEC_PATH` env var to point Databricks CLI to the
Terraform binary. The CLI only uses it if `DATABRICKS_TF_VERSION`
matches the currently used terraform version.
- Use `DATABRICKS_TF_CLI_CONFIG_FILE` env var to point Terraform CLI
config that points to the filesystem mirror for the Databricks provider.
The CLI only uses it if `DATABRICKS_TF_PROVIDER_VERSION` matches the
currently used provider version.


Relevant PR on the VSCode extension side:
https://github.com/databricks/databricks-vscode/pull/1147

Example output of the `databricks bundle debug terraform`:
```
Terraform version: 1.5.5
Terraform URL: https://releases.hashicorp.com/terraform/1.5.5

Databricks Terraform Provider version: 1.38.0
Databricks Terraform Provider URL: https://github.com/databricks/terraform-provider-databricks/releases/tag/v1.38.0

Databricks CLI downloads its Terraform dependencies automatically.

If you run the CLI in an air-gapped environment, you can download the dependencies manually and set these environment variables:

  DATABRICKS_TF_VERSION=1.5.5
  DATABRICKS_TF_EXEC_PATH=/path/to/terraform/binary
  DATABRICKS_TF_PROVIDER_VERSION=1.38.0
  DATABRICKS_TF_CLI_CONFIG_FILE=/path/to/terraform/cli/config.tfrc

Here is an example *.tfrc configuration file:

  disable_checkpoint = true
  provider_installation {
    filesystem_mirror {
      path = "/path/to/a/folder/with/databricks/terraform/provider"
    }
  }

The filesystem mirror path should point to the folder with the Databricks Terraform Provider. The folder should have this structure: /registry.terraform.io/databricks/databricks/terraform-provider-databricks_1.38.0_ARCH.zip

For more information about filesystem mirrors, see the Terraform documentation: https://developer.hashicorp.com/terraform/cli/config/config-file#filesystem_mirror
```

---------

Co-authored-by: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
This commit is contained in:
Ilia Babanov 2024-04-02 14:56:27 +02:00 committed by GitHub
parent 56e393c743
commit 079c416f8d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 317 additions and 7 deletions

View File

@ -12,10 +12,10 @@ import (
"github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/internal/tf/schema"
"github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/env"
"github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/log"
"github.com/hashicorp/go-version"
"github.com/hashicorp/hc-install/product" "github.com/hashicorp/hc-install/product"
"github.com/hashicorp/hc-install/releases" "github.com/hashicorp/hc-install/releases"
"github.com/hashicorp/terraform-exec/tfexec" "github.com/hashicorp/terraform-exec/tfexec"
@ -40,6 +40,17 @@ func (m *initialize) findExecPath(ctx context.Context, b *bundle.Bundle, tf *con
return tf.ExecPath, nil return tf.ExecPath, nil
} }
// Load exec path from the environment if it matches the currently used version.
envExecPath, err := getEnvVarWithMatchingVersion(ctx, TerraformExecPathEnv, TerraformVersionEnv, TerraformVersion.String())
if err != nil {
return "", err
}
if envExecPath != "" {
tf.ExecPath = envExecPath
log.Debugf(ctx, "Using Terraform from %s at %s", TerraformExecPathEnv, tf.ExecPath)
return tf.ExecPath, nil
}
binDir, err := b.CacheDir(context.Background(), "bin") binDir, err := b.CacheDir(context.Background(), "bin")
if err != nil { if err != nil {
return "", err return "", err
@ -60,7 +71,7 @@ func (m *initialize) findExecPath(ctx context.Context, b *bundle.Bundle, tf *con
// Download Terraform to private bin directory. // Download Terraform to private bin directory.
installer := &releases.ExactVersion{ installer := &releases.ExactVersion{
Product: product.Terraform, Product: product.Terraform,
Version: version.Must(version.NewVersion("1.5.5")), Version: TerraformVersion,
InstallDir: binDir, InstallDir: binDir,
Timeout: 1 * time.Minute, Timeout: 1 * time.Minute,
} }
@ -98,14 +109,55 @@ func inheritEnvVars(ctx context.Context, environ map[string]string) error {
} }
// Include $TF_CLI_CONFIG_FILE to override terraform provider in development. // Include $TF_CLI_CONFIG_FILE to override terraform provider in development.
configFile, ok := env.Lookup(ctx, "TF_CLI_CONFIG_FILE") // See: https://developer.hashicorp.com/terraform/cli/config/config-file#explicit-installation-method-configuration
devConfigFile, ok := env.Lookup(ctx, "TF_CLI_CONFIG_FILE")
if ok { if ok {
environ["TF_CLI_CONFIG_FILE"] = devConfigFile
}
// Map $DATABRICKS_TF_CLI_CONFIG_FILE to $TF_CLI_CONFIG_FILE
// VSCode extension provides a file with the "provider_installation.filesystem_mirror" configuration.
// We only use it if the provider version matches the currently used version,
// otherwise terraform will fail to download the right version (even with unrestricted internet access).
configFile, err := getEnvVarWithMatchingVersion(ctx, TerraformCliConfigPathEnv, TerraformProviderVersionEnv, schema.ProviderVersion)
if err != nil {
return err
}
if configFile != "" {
log.Debugf(ctx, "Using Terraform CLI config from %s at %s", TerraformCliConfigPathEnv, configFile)
environ["TF_CLI_CONFIG_FILE"] = configFile environ["TF_CLI_CONFIG_FILE"] = configFile
} }
return nil return nil
} }
// Example: this function will return a value of TF_EXEC_PATH only if the path exists and if TF_VERSION matches the TerraformVersion.
// This function is used for env vars set by the Databricks VSCode extension. The variables are intended to be used by the CLI
// bundled with the Databricks VSCode extension, but users can use different CLI versions in the VSCode terminals, in which case we want to ignore
// the variables if that CLI uses different versions of the dependencies.
func getEnvVarWithMatchingVersion(ctx context.Context, envVarName string, versionVarName string, currentVersion string) (string, error) {
envValue := env.Get(ctx, envVarName)
versionValue := env.Get(ctx, versionVarName)
if envValue == "" || versionValue == "" {
log.Debugf(ctx, "%s and %s aren't defined", envVarName, versionVarName)
return "", nil
}
if versionValue != currentVersion {
log.Debugf(ctx, "%s as %s does not match the current version %s, ignoring %s", versionVarName, versionValue, currentVersion, envVarName)
return "", nil
}
_, err := os.Stat(envValue)
if err != nil {
if os.IsNotExist(err) {
log.Debugf(ctx, "%s at %s does not exist, ignoring %s", envVarName, envValue, versionVarName)
return "", nil
} else {
return "", err
}
}
return envValue, nil
}
// This function sets temp dir location for terraform to use. If user does not // This function sets temp dir location for terraform to use. If user does not
// specify anything here, we fall back to a `tmp` directory in the bundle's cache // specify anything here, we fall back to a `tmp` directory in the bundle's cache
// directory // directory

View File

@ -4,12 +4,16 @@ import (
"context" "context"
"os" "os"
"os/exec" "os/exec"
"path/filepath"
"runtime" "runtime"
"strings" "strings"
"testing" "testing"
"github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/internal/tf/schema"
"github.com/databricks/cli/libs/env"
"github.com/hashicorp/hc-install/product"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"golang.org/x/exp/maps" "golang.org/x/exp/maps"
@ -269,3 +273,122 @@ func TestSetUserProfileFromInheritEnvVars(t *testing.T) {
assert.Contains(t, env, "USERPROFILE") assert.Contains(t, env, "USERPROFILE")
assert.Equal(t, env["USERPROFILE"], "c:\\foo\\c") assert.Equal(t, env["USERPROFILE"], "c:\\foo\\c")
} }
func TestInheritEnvVarsWithAbsentTFConfigFile(t *testing.T) {
ctx := context.Background()
envMap := map[string]string{}
ctx = env.Set(ctx, "DATABRICKS_TF_PROVIDER_VERSION", schema.ProviderVersion)
ctx = env.Set(ctx, "DATABRICKS_TF_CLI_CONFIG_FILE", "/tmp/config.tfrc")
err := inheritEnvVars(ctx, envMap)
require.NoError(t, err)
require.NotContains(t, envMap, "TF_CLI_CONFIG_FILE")
}
func TestInheritEnvVarsWithWrongTFProviderVersion(t *testing.T) {
ctx := context.Background()
envMap := map[string]string{}
configFile := createTempFile(t, t.TempDir(), "config.tfrc", false)
ctx = env.Set(ctx, "DATABRICKS_TF_PROVIDER_VERSION", "wrong")
ctx = env.Set(ctx, "DATABRICKS_TF_CLI_CONFIG_FILE", configFile)
err := inheritEnvVars(ctx, envMap)
require.NoError(t, err)
require.NotContains(t, envMap, "TF_CLI_CONFIG_FILE")
}
func TestInheritEnvVarsWithCorrectTFCLIConfigFile(t *testing.T) {
ctx := context.Background()
envMap := map[string]string{}
configFile := createTempFile(t, t.TempDir(), "config.tfrc", false)
ctx = env.Set(ctx, "DATABRICKS_TF_PROVIDER_VERSION", schema.ProviderVersion)
ctx = env.Set(ctx, "DATABRICKS_TF_CLI_CONFIG_FILE", configFile)
err := inheritEnvVars(ctx, envMap)
require.NoError(t, err)
require.Contains(t, envMap, "TF_CLI_CONFIG_FILE")
require.Equal(t, configFile, envMap["TF_CLI_CONFIG_FILE"])
}
func TestFindExecPathFromEnvironmentWithWrongVersion(t *testing.T) {
ctx := context.Background()
m := &initialize{}
b := &bundle.Bundle{
RootPath: t.TempDir(),
Config: config.Root{
Bundle: config.Bundle{
Target: "whatever",
Terraform: &config.Terraform{},
},
},
}
// Create a pre-existing terraform bin to avoid downloading it
cacheDir, _ := b.CacheDir(ctx, "bin")
existingExecPath := createTempFile(t, cacheDir, product.Terraform.BinaryName(), true)
// Create a new terraform binary and expose it through env vars
tmpBinPath := createTempFile(t, t.TempDir(), "terraform-bin", true)
ctx = env.Set(ctx, "DATABRICKS_TF_VERSION", "1.2.3")
ctx = env.Set(ctx, "DATABRICKS_TF_EXEC_PATH", tmpBinPath)
_, err := m.findExecPath(ctx, b, b.Config.Bundle.Terraform)
require.NoError(t, err)
require.Equal(t, existingExecPath, b.Config.Bundle.Terraform.ExecPath)
}
func TestFindExecPathFromEnvironmentWithCorrectVersionAndNoBinary(t *testing.T) {
ctx := context.Background()
m := &initialize{}
b := &bundle.Bundle{
RootPath: t.TempDir(),
Config: config.Root{
Bundle: config.Bundle{
Target: "whatever",
Terraform: &config.Terraform{},
},
},
}
// Create a pre-existing terraform bin to avoid downloading it
cacheDir, _ := b.CacheDir(ctx, "bin")
existingExecPath := createTempFile(t, cacheDir, product.Terraform.BinaryName(), true)
ctx = env.Set(ctx, "DATABRICKS_TF_VERSION", TerraformVersion.String())
ctx = env.Set(ctx, "DATABRICKS_TF_EXEC_PATH", "/tmp/terraform")
_, err := m.findExecPath(ctx, b, b.Config.Bundle.Terraform)
require.NoError(t, err)
require.Equal(t, existingExecPath, b.Config.Bundle.Terraform.ExecPath)
}
func TestFindExecPathFromEnvironmentWithCorrectVersionAndBinary(t *testing.T) {
ctx := context.Background()
m := &initialize{}
b := &bundle.Bundle{
RootPath: t.TempDir(),
Config: config.Root{
Bundle: config.Bundle{
Target: "whatever",
Terraform: &config.Terraform{},
},
},
}
// Create a pre-existing terraform bin to avoid downloading it
cacheDir, _ := b.CacheDir(ctx, "bin")
createTempFile(t, cacheDir, product.Terraform.BinaryName(), true)
// Create a new terraform binary and expose it through env vars
tmpBinPath := createTempFile(t, t.TempDir(), "terraform-bin", true)
ctx = env.Set(ctx, "DATABRICKS_TF_VERSION", TerraformVersion.String())
ctx = env.Set(ctx, "DATABRICKS_TF_EXEC_PATH", tmpBinPath)
_, err := m.findExecPath(ctx, b, b.Config.Bundle.Terraform)
require.NoError(t, err)
require.Equal(t, tmpBinPath, b.Config.Bundle.Terraform.ExecPath)
}
func createTempFile(t *testing.T, dest string, name string, executable bool) string {
binPath := filepath.Join(dest, name)
f, err := os.Create(binPath)
require.NoError(t, err)
defer func() {
err = f.Close()
require.NoError(t, err)
}()
if executable {
err = f.Chmod(0777)
require.NoError(t, err)
}
return binPath
}

View File

@ -1,4 +1,34 @@
package terraform package terraform
import (
"github.com/databricks/cli/bundle/internal/tf/schema"
"github.com/hashicorp/go-version"
)
const TerraformStateFileName = "terraform.tfstate" const TerraformStateFileName = "terraform.tfstate"
const TerraformConfigFileName = "bundle.tf.json" const TerraformConfigFileName = "bundle.tf.json"
// Users can provide their own terraform binary and databricks terraform provider by setting the following environment variables.
// This allows users to use the CLI in an air-gapped environments. See the `debug terraform` command.
const TerraformExecPathEnv = "DATABRICKS_TF_EXEC_PATH"
const TerraformVersionEnv = "DATABRICKS_TF_VERSION"
const TerraformCliConfigPathEnv = "DATABRICKS_TF_CLI_CONFIG_FILE"
const TerraformProviderVersionEnv = "DATABRICKS_TF_PROVIDER_VERSION"
var TerraformVersion = version.Must(version.NewVersion("1.5.5"))
type TerraformMetadata struct {
Version string `json:"version"`
ProviderHost string `json:"providerHost"`
ProviderSource string `json:"providerSource"`
ProviderVersion string `json:"providerVersion"`
}
func NewTerraformMetadata() *TerraformMetadata {
return &TerraformMetadata{
Version: TerraformVersion.String(),
ProviderHost: schema.ProviderHost,
ProviderSource: schema.ProviderSource,
ProviderVersion: schema.ProviderVersion,
}
}

View File

@ -19,13 +19,17 @@ type Root struct {
Resource *Resources `json:"resource,omitempty"` Resource *Resources `json:"resource,omitempty"`
} }
const ProviderHost = "registry.terraform.io"
const ProviderSource = "databricks/databricks"
const ProviderVersion = "{{ .ProviderVersion }}"
func NewRoot() *Root { func NewRoot() *Root {
return &Root{ return &Root{
Terraform: map[string]interface{}{ Terraform: map[string]interface{}{
"required_providers": map[string]interface{}{ "required_providers": map[string]interface{}{
"databricks": map[string]interface{}{ "databricks": map[string]interface{}{
"source": "databricks/databricks", "source": ProviderSource,
"version": "{{ .ProviderVersion }}", "version": ProviderVersion,
}, },
}, },
}, },

View File

@ -19,13 +19,17 @@ type Root struct {
Resource *Resources `json:"resource,omitempty"` Resource *Resources `json:"resource,omitempty"`
} }
const ProviderHost = "registry.terraform.io"
const ProviderSource = "databricks/databricks"
const ProviderVersion = "1.38.0"
func NewRoot() *Root { func NewRoot() *Root {
return &Root{ return &Root{
Terraform: map[string]interface{}{ Terraform: map[string]interface{}{
"required_providers": map[string]interface{}{ "required_providers": map[string]interface{}{
"databricks": map[string]interface{}{ "databricks": map[string]interface{}{
"source": "databricks/databricks", "source": ProviderSource,
"version": "1.38.0", "version": ProviderVersion,
}, },
}, },
}, },

View File

@ -25,6 +25,7 @@ func New() *cobra.Command {
cmd.AddCommand(newInitCommand()) cmd.AddCommand(newInitCommand())
cmd.AddCommand(newSummaryCommand()) cmd.AddCommand(newSummaryCommand())
cmd.AddCommand(newGenerateCommand()) cmd.AddCommand(newGenerateCommand())
cmd.AddCommand(newDebugCommand())
cmd.AddCommand(deployment.NewDeploymentCommand()) cmd.AddCommand(deployment.NewDeploymentCommand())
return cmd return cmd
} }

18
cmd/bundle/debug.go Normal file
View File

@ -0,0 +1,18 @@
package bundle
import (
"github.com/databricks/cli/cmd/bundle/debug"
"github.com/spf13/cobra"
)
func newDebugCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "debug",
Short: "Debug information about bundles",
Long: "Debug information about bundles",
// This command group is currently intended for the Databricks VSCode extension only
Hidden: true,
}
cmd.AddCommand(debug.NewTerraformCommand())
return cmd
}

View File

@ -0,0 +1,78 @@
package debug
import (
"encoding/json"
"fmt"
"github.com/databricks/cli/bundle/deploy/terraform"
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/flags"
"github.com/spf13/cobra"
)
type Dependencies struct {
Terraform *terraform.TerraformMetadata `json:"terraform"`
}
func NewTerraformCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "terraform",
Short: "Prints Terraform dependencies required for the bundle commands",
Args: root.NoArgs,
Annotations: map[string]string{
"template": `Terraform version: {{.Version}}
Terraform URL: https://releases.hashicorp.com/terraform/{{.Version}}
Databricks Terraform Provider version: {{.ProviderVersion}}
Databricks Terraform Provider URL: https://github.com/databricks/terraform-provider-databricks/releases/tag/v{{.ProviderVersion}}
Databricks CLI downloads its Terraform dependencies automatically.
If you run the CLI in an air-gapped environment, you can download the dependencies manually and set these environment variables:
DATABRICKS_TF_VERSION={{.Version}}
DATABRICKS_TF_EXEC_PATH=/path/to/terraform/binary
DATABRICKS_TF_PROVIDER_VERSION={{.ProviderVersion}}
DATABRICKS_TF_CLI_CONFIG_FILE=/path/to/terraform/cli/config.tfrc
Here is an example *.tfrc configuration file:
disable_checkpoint = true
provider_installation {
filesystem_mirror {
path = "/path/to/a/folder/with/databricks/terraform/provider"
}
}
The filesystem mirror path should point to the folder with the Databricks Terraform Provider. The folder should have this structure: /{{.ProviderHost}}/{{.ProviderSource}}/terraform-provider-databricks_{{.ProviderVersion}}_ARCH.zip
For more information about filesystem mirrors, see the Terraform documentation: https://developer.hashicorp.com/terraform/cli/config/config-file#filesystem_mirror
`,
},
// This command is currently intended for the Databricks VSCode extension only
Hidden: true,
}
cmd.RunE = func(cmd *cobra.Command, args []string) error {
dependencies := &Dependencies{
Terraform: terraform.NewTerraformMetadata(),
}
switch root.OutputType(cmd) {
case flags.OutputText:
cmdio.Render(cmd.Context(), dependencies.Terraform)
case flags.OutputJSON:
buf, err := json.MarshalIndent(dependencies, "", " ")
if err != nil {
return err
}
cmd.OutOrStdout().Write(buf)
default:
return fmt.Errorf("unknown output type %s", root.OutputType(cmd))
}
return nil
}
return cmd
}