From 079c416f8d0e4600901c3704304a52c140a862eb Mon Sep 17 00:00:00 2001 From: Ilia Babanov Date: Tue, 2 Apr 2024 14:56:27 +0200 Subject: [PATCH] Add `bundle debug terraform` command (#1294) - Add `bundle debug terraform` command. It prints versions of the Terraform and the Databricks Terraform provider. In the text mode it also explains how to setup the CLI in environments with restricted internet access. - Use `DATABRICKS_TF_EXEC_PATH` env var to point Databricks CLI to the Terraform binary. The CLI only uses it if `DATABRICKS_TF_VERSION` matches the currently used terraform version. - Use `DATABRICKS_TF_CLI_CONFIG_FILE` env var to point Terraform CLI config that points to the filesystem mirror for the Databricks provider. The CLI only uses it if `DATABRICKS_TF_PROVIDER_VERSION` matches the currently used provider version. Relevant PR on the VSCode extension side: https://github.com/databricks/databricks-vscode/pull/1147 Example output of the `databricks bundle debug terraform`: ``` Terraform version: 1.5.5 Terraform URL: https://releases.hashicorp.com/terraform/1.5.5 Databricks Terraform Provider version: 1.38.0 Databricks Terraform Provider URL: https://github.com/databricks/terraform-provider-databricks/releases/tag/v1.38.0 Databricks CLI downloads its Terraform dependencies automatically. If you run the CLI in an air-gapped environment, you can download the dependencies manually and set these environment variables: DATABRICKS_TF_VERSION=1.5.5 DATABRICKS_TF_EXEC_PATH=/path/to/terraform/binary DATABRICKS_TF_PROVIDER_VERSION=1.38.0 DATABRICKS_TF_CLI_CONFIG_FILE=/path/to/terraform/cli/config.tfrc Here is an example *.tfrc configuration file: disable_checkpoint = true provider_installation { filesystem_mirror { path = "/path/to/a/folder/with/databricks/terraform/provider" } } The filesystem mirror path should point to the folder with the Databricks Terraform Provider. The folder should have this structure: /registry.terraform.io/databricks/databricks/terraform-provider-databricks_1.38.0_ARCH.zip For more information about filesystem mirrors, see the Terraform documentation: https://developer.hashicorp.com/terraform/cli/config/config-file#filesystem_mirror ``` --------- Co-authored-by: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> --- bundle/deploy/terraform/init.go | 58 ++++++++- bundle/deploy/terraform/init_test.go | 123 ++++++++++++++++++ bundle/deploy/terraform/pkg.go | 30 +++++ .../tf/codegen/templates/root.go.tmpl | 8 +- bundle/internal/tf/schema/root.go | 8 +- cmd/bundle/bundle.go | 1 + cmd/bundle/debug.go | 18 +++ cmd/bundle/debug/terraform.go | 78 +++++++++++ 8 files changed, 317 insertions(+), 7 deletions(-) create mode 100644 cmd/bundle/debug.go create mode 100644 cmd/bundle/debug/terraform.go diff --git a/bundle/deploy/terraform/init.go b/bundle/deploy/terraform/init.go index ca1fc8ca..9f423531 100644 --- a/bundle/deploy/terraform/init.go +++ b/bundle/deploy/terraform/init.go @@ -12,10 +12,10 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/internal/tf/schema" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" - "github.com/hashicorp/go-version" "github.com/hashicorp/hc-install/product" "github.com/hashicorp/hc-install/releases" "github.com/hashicorp/terraform-exec/tfexec" @@ -40,6 +40,17 @@ func (m *initialize) findExecPath(ctx context.Context, b *bundle.Bundle, tf *con return tf.ExecPath, nil } + // Load exec path from the environment if it matches the currently used version. + envExecPath, err := getEnvVarWithMatchingVersion(ctx, TerraformExecPathEnv, TerraformVersionEnv, TerraformVersion.String()) + if err != nil { + return "", err + } + if envExecPath != "" { + tf.ExecPath = envExecPath + log.Debugf(ctx, "Using Terraform from %s at %s", TerraformExecPathEnv, tf.ExecPath) + return tf.ExecPath, nil + } + binDir, err := b.CacheDir(context.Background(), "bin") if err != nil { return "", err @@ -60,7 +71,7 @@ func (m *initialize) findExecPath(ctx context.Context, b *bundle.Bundle, tf *con // Download Terraform to private bin directory. installer := &releases.ExactVersion{ Product: product.Terraform, - Version: version.Must(version.NewVersion("1.5.5")), + Version: TerraformVersion, InstallDir: binDir, Timeout: 1 * time.Minute, } @@ -98,14 +109,55 @@ func inheritEnvVars(ctx context.Context, environ map[string]string) error { } // Include $TF_CLI_CONFIG_FILE to override terraform provider in development. - configFile, ok := env.Lookup(ctx, "TF_CLI_CONFIG_FILE") + // See: https://developer.hashicorp.com/terraform/cli/config/config-file#explicit-installation-method-configuration + devConfigFile, ok := env.Lookup(ctx, "TF_CLI_CONFIG_FILE") if ok { + environ["TF_CLI_CONFIG_FILE"] = devConfigFile + } + + // Map $DATABRICKS_TF_CLI_CONFIG_FILE to $TF_CLI_CONFIG_FILE + // VSCode extension provides a file with the "provider_installation.filesystem_mirror" configuration. + // We only use it if the provider version matches the currently used version, + // otherwise terraform will fail to download the right version (even with unrestricted internet access). + configFile, err := getEnvVarWithMatchingVersion(ctx, TerraformCliConfigPathEnv, TerraformProviderVersionEnv, schema.ProviderVersion) + if err != nil { + return err + } + if configFile != "" { + log.Debugf(ctx, "Using Terraform CLI config from %s at %s", TerraformCliConfigPathEnv, configFile) environ["TF_CLI_CONFIG_FILE"] = configFile } return nil } +// Example: this function will return a value of TF_EXEC_PATH only if the path exists and if TF_VERSION matches the TerraformVersion. +// This function is used for env vars set by the Databricks VSCode extension. The variables are intended to be used by the CLI +// bundled with the Databricks VSCode extension, but users can use different CLI versions in the VSCode terminals, in which case we want to ignore +// the variables if that CLI uses different versions of the dependencies. +func getEnvVarWithMatchingVersion(ctx context.Context, envVarName string, versionVarName string, currentVersion string) (string, error) { + envValue := env.Get(ctx, envVarName) + versionValue := env.Get(ctx, versionVarName) + if envValue == "" || versionValue == "" { + log.Debugf(ctx, "%s and %s aren't defined", envVarName, versionVarName) + return "", nil + } + if versionValue != currentVersion { + log.Debugf(ctx, "%s as %s does not match the current version %s, ignoring %s", versionVarName, versionValue, currentVersion, envVarName) + return "", nil + } + _, err := os.Stat(envValue) + if err != nil { + if os.IsNotExist(err) { + log.Debugf(ctx, "%s at %s does not exist, ignoring %s", envVarName, envValue, versionVarName) + return "", nil + } else { + return "", err + } + } + return envValue, nil +} + // This function sets temp dir location for terraform to use. If user does not // specify anything here, we fall back to a `tmp` directory in the bundle's cache // directory diff --git a/bundle/deploy/terraform/init_test.go b/bundle/deploy/terraform/init_test.go index 29bd80a3..ece89719 100644 --- a/bundle/deploy/terraform/init_test.go +++ b/bundle/deploy/terraform/init_test.go @@ -4,12 +4,16 @@ import ( "context" "os" "os/exec" + "path/filepath" "runtime" "strings" "testing" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/env" + "github.com/hashicorp/hc-install/product" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/exp/maps" @@ -269,3 +273,122 @@ func TestSetUserProfileFromInheritEnvVars(t *testing.T) { assert.Contains(t, env, "USERPROFILE") assert.Equal(t, env["USERPROFILE"], "c:\\foo\\c") } + +func TestInheritEnvVarsWithAbsentTFConfigFile(t *testing.T) { + ctx := context.Background() + envMap := map[string]string{} + ctx = env.Set(ctx, "DATABRICKS_TF_PROVIDER_VERSION", schema.ProviderVersion) + ctx = env.Set(ctx, "DATABRICKS_TF_CLI_CONFIG_FILE", "/tmp/config.tfrc") + err := inheritEnvVars(ctx, envMap) + require.NoError(t, err) + require.NotContains(t, envMap, "TF_CLI_CONFIG_FILE") +} + +func TestInheritEnvVarsWithWrongTFProviderVersion(t *testing.T) { + ctx := context.Background() + envMap := map[string]string{} + configFile := createTempFile(t, t.TempDir(), "config.tfrc", false) + ctx = env.Set(ctx, "DATABRICKS_TF_PROVIDER_VERSION", "wrong") + ctx = env.Set(ctx, "DATABRICKS_TF_CLI_CONFIG_FILE", configFile) + err := inheritEnvVars(ctx, envMap) + require.NoError(t, err) + require.NotContains(t, envMap, "TF_CLI_CONFIG_FILE") +} + +func TestInheritEnvVarsWithCorrectTFCLIConfigFile(t *testing.T) { + ctx := context.Background() + envMap := map[string]string{} + configFile := createTempFile(t, t.TempDir(), "config.tfrc", false) + ctx = env.Set(ctx, "DATABRICKS_TF_PROVIDER_VERSION", schema.ProviderVersion) + ctx = env.Set(ctx, "DATABRICKS_TF_CLI_CONFIG_FILE", configFile) + err := inheritEnvVars(ctx, envMap) + require.NoError(t, err) + require.Contains(t, envMap, "TF_CLI_CONFIG_FILE") + require.Equal(t, configFile, envMap["TF_CLI_CONFIG_FILE"]) +} + +func TestFindExecPathFromEnvironmentWithWrongVersion(t *testing.T) { + ctx := context.Background() + m := &initialize{} + b := &bundle.Bundle{ + RootPath: t.TempDir(), + Config: config.Root{ + Bundle: config.Bundle{ + Target: "whatever", + Terraform: &config.Terraform{}, + }, + }, + } + // Create a pre-existing terraform bin to avoid downloading it + cacheDir, _ := b.CacheDir(ctx, "bin") + existingExecPath := createTempFile(t, cacheDir, product.Terraform.BinaryName(), true) + // Create a new terraform binary and expose it through env vars + tmpBinPath := createTempFile(t, t.TempDir(), "terraform-bin", true) + ctx = env.Set(ctx, "DATABRICKS_TF_VERSION", "1.2.3") + ctx = env.Set(ctx, "DATABRICKS_TF_EXEC_PATH", tmpBinPath) + _, err := m.findExecPath(ctx, b, b.Config.Bundle.Terraform) + require.NoError(t, err) + require.Equal(t, existingExecPath, b.Config.Bundle.Terraform.ExecPath) +} + +func TestFindExecPathFromEnvironmentWithCorrectVersionAndNoBinary(t *testing.T) { + ctx := context.Background() + m := &initialize{} + b := &bundle.Bundle{ + RootPath: t.TempDir(), + Config: config.Root{ + Bundle: config.Bundle{ + Target: "whatever", + Terraform: &config.Terraform{}, + }, + }, + } + // Create a pre-existing terraform bin to avoid downloading it + cacheDir, _ := b.CacheDir(ctx, "bin") + existingExecPath := createTempFile(t, cacheDir, product.Terraform.BinaryName(), true) + + ctx = env.Set(ctx, "DATABRICKS_TF_VERSION", TerraformVersion.String()) + ctx = env.Set(ctx, "DATABRICKS_TF_EXEC_PATH", "/tmp/terraform") + _, err := m.findExecPath(ctx, b, b.Config.Bundle.Terraform) + require.NoError(t, err) + require.Equal(t, existingExecPath, b.Config.Bundle.Terraform.ExecPath) +} + +func TestFindExecPathFromEnvironmentWithCorrectVersionAndBinary(t *testing.T) { + ctx := context.Background() + m := &initialize{} + b := &bundle.Bundle{ + RootPath: t.TempDir(), + Config: config.Root{ + Bundle: config.Bundle{ + Target: "whatever", + Terraform: &config.Terraform{}, + }, + }, + } + // Create a pre-existing terraform bin to avoid downloading it + cacheDir, _ := b.CacheDir(ctx, "bin") + createTempFile(t, cacheDir, product.Terraform.BinaryName(), true) + // Create a new terraform binary and expose it through env vars + tmpBinPath := createTempFile(t, t.TempDir(), "terraform-bin", true) + ctx = env.Set(ctx, "DATABRICKS_TF_VERSION", TerraformVersion.String()) + ctx = env.Set(ctx, "DATABRICKS_TF_EXEC_PATH", tmpBinPath) + _, err := m.findExecPath(ctx, b, b.Config.Bundle.Terraform) + require.NoError(t, err) + require.Equal(t, tmpBinPath, b.Config.Bundle.Terraform.ExecPath) +} + +func createTempFile(t *testing.T, dest string, name string, executable bool) string { + binPath := filepath.Join(dest, name) + f, err := os.Create(binPath) + require.NoError(t, err) + defer func() { + err = f.Close() + require.NoError(t, err) + }() + if executable { + err = f.Chmod(0777) + require.NoError(t, err) + } + return binPath +} diff --git a/bundle/deploy/terraform/pkg.go b/bundle/deploy/terraform/pkg.go index 2d9293d1..911583f2 100644 --- a/bundle/deploy/terraform/pkg.go +++ b/bundle/deploy/terraform/pkg.go @@ -1,4 +1,34 @@ package terraform +import ( + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/hashicorp/go-version" +) + const TerraformStateFileName = "terraform.tfstate" const TerraformConfigFileName = "bundle.tf.json" + +// Users can provide their own terraform binary and databricks terraform provider by setting the following environment variables. +// This allows users to use the CLI in an air-gapped environments. See the `debug terraform` command. +const TerraformExecPathEnv = "DATABRICKS_TF_EXEC_PATH" +const TerraformVersionEnv = "DATABRICKS_TF_VERSION" +const TerraformCliConfigPathEnv = "DATABRICKS_TF_CLI_CONFIG_FILE" +const TerraformProviderVersionEnv = "DATABRICKS_TF_PROVIDER_VERSION" + +var TerraformVersion = version.Must(version.NewVersion("1.5.5")) + +type TerraformMetadata struct { + Version string `json:"version"` + ProviderHost string `json:"providerHost"` + ProviderSource string `json:"providerSource"` + ProviderVersion string `json:"providerVersion"` +} + +func NewTerraformMetadata() *TerraformMetadata { + return &TerraformMetadata{ + Version: TerraformVersion.String(), + ProviderHost: schema.ProviderHost, + ProviderSource: schema.ProviderSource, + ProviderVersion: schema.ProviderVersion, + } +} diff --git a/bundle/internal/tf/codegen/templates/root.go.tmpl b/bundle/internal/tf/codegen/templates/root.go.tmpl index 57fa7129..e03e978f 100644 --- a/bundle/internal/tf/codegen/templates/root.go.tmpl +++ b/bundle/internal/tf/codegen/templates/root.go.tmpl @@ -19,13 +19,17 @@ type Root struct { Resource *Resources `json:"resource,omitempty"` } +const ProviderHost = "registry.terraform.io" +const ProviderSource = "databricks/databricks" +const ProviderVersion = "{{ .ProviderVersion }}" + func NewRoot() *Root { return &Root{ Terraform: map[string]interface{}{ "required_providers": map[string]interface{}{ "databricks": map[string]interface{}{ - "source": "databricks/databricks", - "version": "{{ .ProviderVersion }}", + "source": ProviderSource, + "version": ProviderVersion, }, }, }, diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go index 118e2857..39532632 100644 --- a/bundle/internal/tf/schema/root.go +++ b/bundle/internal/tf/schema/root.go @@ -19,13 +19,17 @@ type Root struct { Resource *Resources `json:"resource,omitempty"` } +const ProviderHost = "registry.terraform.io" +const ProviderSource = "databricks/databricks" +const ProviderVersion = "1.38.0" + func NewRoot() *Root { return &Root{ Terraform: map[string]interface{}{ "required_providers": map[string]interface{}{ "databricks": map[string]interface{}{ - "source": "databricks/databricks", - "version": "1.38.0", + "source": ProviderSource, + "version": ProviderVersion, }, }, }, diff --git a/cmd/bundle/bundle.go b/cmd/bundle/bundle.go index 43a9ef68..1db60d58 100644 --- a/cmd/bundle/bundle.go +++ b/cmd/bundle/bundle.go @@ -25,6 +25,7 @@ func New() *cobra.Command { cmd.AddCommand(newInitCommand()) cmd.AddCommand(newSummaryCommand()) cmd.AddCommand(newGenerateCommand()) + cmd.AddCommand(newDebugCommand()) cmd.AddCommand(deployment.NewDeploymentCommand()) return cmd } diff --git a/cmd/bundle/debug.go b/cmd/bundle/debug.go new file mode 100644 index 00000000..42d16eab --- /dev/null +++ b/cmd/bundle/debug.go @@ -0,0 +1,18 @@ +package bundle + +import ( + "github.com/databricks/cli/cmd/bundle/debug" + "github.com/spf13/cobra" +) + +func newDebugCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "debug", + Short: "Debug information about bundles", + Long: "Debug information about bundles", + // This command group is currently intended for the Databricks VSCode extension only + Hidden: true, + } + cmd.AddCommand(debug.NewTerraformCommand()) + return cmd +} diff --git a/cmd/bundle/debug/terraform.go b/cmd/bundle/debug/terraform.go new file mode 100644 index 00000000..843ecac4 --- /dev/null +++ b/cmd/bundle/debug/terraform.go @@ -0,0 +1,78 @@ +package debug + +import ( + "encoding/json" + "fmt" + + "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/spf13/cobra" +) + +type Dependencies struct { + Terraform *terraform.TerraformMetadata `json:"terraform"` +} + +func NewTerraformCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "terraform", + Short: "Prints Terraform dependencies required for the bundle commands", + Args: root.NoArgs, + Annotations: map[string]string{ + "template": `Terraform version: {{.Version}} +Terraform URL: https://releases.hashicorp.com/terraform/{{.Version}} + +Databricks Terraform Provider version: {{.ProviderVersion}} +Databricks Terraform Provider URL: https://github.com/databricks/terraform-provider-databricks/releases/tag/v{{.ProviderVersion}} + +Databricks CLI downloads its Terraform dependencies automatically. + +If you run the CLI in an air-gapped environment, you can download the dependencies manually and set these environment variables: + + DATABRICKS_TF_VERSION={{.Version}} + DATABRICKS_TF_EXEC_PATH=/path/to/terraform/binary + DATABRICKS_TF_PROVIDER_VERSION={{.ProviderVersion}} + DATABRICKS_TF_CLI_CONFIG_FILE=/path/to/terraform/cli/config.tfrc + +Here is an example *.tfrc configuration file: + + disable_checkpoint = true + provider_installation { + filesystem_mirror { + path = "/path/to/a/folder/with/databricks/terraform/provider" + } + } + +The filesystem mirror path should point to the folder with the Databricks Terraform Provider. The folder should have this structure: /{{.ProviderHost}}/{{.ProviderSource}}/terraform-provider-databricks_{{.ProviderVersion}}_ARCH.zip + +For more information about filesystem mirrors, see the Terraform documentation: https://developer.hashicorp.com/terraform/cli/config/config-file#filesystem_mirror +`, + }, + // This command is currently intended for the Databricks VSCode extension only + Hidden: true, + } + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + dependencies := &Dependencies{ + Terraform: terraform.NewTerraformMetadata(), + } + switch root.OutputType(cmd) { + case flags.OutputText: + cmdio.Render(cmd.Context(), dependencies.Terraform) + case flags.OutputJSON: + buf, err := json.MarshalIndent(dependencies, "", " ") + if err != nil { + return err + } + cmd.OutOrStdout().Write(buf) + default: + return fmt.Errorf("unknown output type %s", root.OutputType(cmd)) + } + + return nil + } + + return cmd +}