Use real terraform in acceptance tests (#2267)

## Changes
- Add a script install_terraform.py that downloads terraform and
provider and generates a config to use, inspired by
https://gist.github.com/pietern/1cb6b6f3e0a452328e13cdc75031105e
- Make acceptance tests run this script once before running the tests
and set the required env vars to make cli use this terraform
installation.
- Use OS-specific directory for things that are build by acceptance test
runner (CLI and terraform).

This enables acceptance tests against cloud #2242 and local test for
bundle deploy #2254.

## Tests
- Add an acceptance test for standalone terraform. This is useful to
debug terraform with TF_LOG=DEBUG to see that it uses local provider.
- Other acceptance tests are updated with regard to terraform exec path.
- The overall time for tests locally is unchanged (if terraform is
already fetched).

acceptance/.gitignore

@@ -0,0 +1 @@
+build

acceptance/acceptance_test.go

@@ -77,6 +77,11 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	cwd, err := os.Getwd()
 	require.NoError(t, err)
 
+	buildDir := filepath.Join(cwd, "build", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH))
+
+	// Download terraform and provider and create config; this also creates build directory.
+	RunCommand(t, []string{"python3", filepath.Join(cwd, "install_terraform.py"), "--targetdir", buildDir}, ".")
+
 	coverDir := os.Getenv("CLI_GOCOVERDIR")
 
 	if coverDir != "" {
@@ -93,7 +98,7 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 		t.Setenv("CMD_SERVER_URL", cmdServer.URL)
 		execPath = filepath.Join(cwd, "bin", "callserver.py")
 	} else {
-		execPath = BuildCLI(t, cwd, coverDir)
+		execPath = BuildCLI(t, buildDir, coverDir)
 	}
 
 	t.Setenv("CLI", execPath)
@@ -123,11 +128,24 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 		homeDir := t.TempDir()
 		// Do not read user's ~/.databrickscfg
 		t.Setenv(env.HomeEnvVar(), homeDir)
-
-		// Prevent CLI from downloading terraform in each test:
-		t.Setenv("DATABRICKS_TF_EXEC_PATH", tempHomeDir)
 	}
 
+	terraformrcPath := filepath.Join(buildDir, ".terraformrc")
+	t.Setenv("TF_CLI_CONFIG_FILE", terraformrcPath)
+	t.Setenv("DATABRICKS_TF_CLI_CONFIG_FILE", terraformrcPath)
+	repls.SetPath(terraformrcPath, "$DATABRICKS_TF_CLI_CONFIG_FILE")
+
+	terraformExecPath := filepath.Join(buildDir, "terraform")
+	if runtime.GOOS == "windows" {
+		terraformExecPath += ".exe"
+	}
+	t.Setenv("DATABRICKS_TF_EXEC_PATH", terraformExecPath)
+	t.Setenv("TERRAFORM", terraformExecPath)
+	repls.SetPath(terraformExecPath, "$TERRAFORM")
+
+	// do it last so that full paths match first:
+	repls.SetPath(buildDir, "$BUILD_DIR")
+
 	workspaceClient, err := databricks.NewWorkspaceClient()
 	require.NoError(t, err)
 
@@ -406,13 +424,12 @@ func readMergedScriptContents(t *testing.T, dir string) string {
 	return strings.Join(prepares, "\n")
 }
 
-func BuildCLI(t *testing.T, cwd, coverDir string) string {
-	execPath := filepath.Join(cwd, "build", "databricks")
+func BuildCLI(t *testing.T, buildDir, coverDir string) string {
+	execPath := filepath.Join(buildDir, "databricks")
 	if runtime.GOOS == "windows" {
 		execPath += ".exe"
 	}
 
-	start := time.Now()
 	args := []string{
 		"go", "build",
 		"-mod", "vendor",
@@ -430,20 +447,8 @@ func BuildCLI(t *testing.T, cwd, coverDir string) string {
 		args = append(args, "-buildvcs=false")
 	}
 
-	cmd := exec.Command(args[0], args[1:]...)
-	cmd.Dir = ".."
-	out, err := cmd.CombinedOutput()
-	elapsed := time.Since(start)
-	t.Logf("%s took %s", args, elapsed)
-	require.NoError(t, err, "go build failed: %s: %s\n%s", args, err, out)
-	if len(out) > 0 {
-		t.Logf("go build output: %s: %s", args, out)
-	}
-
-	// Quick check + warm up cache:
-	cmd = exec.Command(execPath, "--version")
-	out, err = cmd.CombinedOutput()
-	require.NoError(t, err, "%s --version failed: %s\n%s", execPath, err, out)
+	RunCommand(t, args, "..")
+	RunCommand(t, []string{execPath, "--version"}, ".")
 	return execPath
 }
 
@@ -581,3 +586,17 @@ func getUVDefaultCacheDir(t *testing.T) string {
 		return cacheDir + "/uv"
 	}
 }
+
+func RunCommand(t *testing.T, args []string, dir string) {
+	start := time.Now()
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Dir = dir
+	out, err := cmd.CombinedOutput()
+	elapsed := time.Since(start)
+	t.Logf("%s took %s", args, elapsed)
+
+	require.NoError(t, err, "%s failed: %s\n%s", args, err, out)
+	if len(out) > 0 {
+		t.Logf("%s output: %s", args, out)
+	}
+}

acceptance/build/.gitignore

@@ -1 +0,0 @@
-databricks

acceptance/bundle/variables/git-branch/output.txt

@@ -11,7 +11,7 @@
     "name": "git",
     "target": "prod",
     "terraform": {
-      "exec_path": "$TMPHOME"
+      "exec_path": "$TERRAFORM"
     }
   },
   "sync": {
@@ -61,7 +61,7 @@ Validation OK!
     "name": "git",
     "target": "dev",
     "terraform": {
-      "exec_path": "$TMPHOME"
+      "exec_path": "$TERRAFORM"
     }
   },
   "sync": {

acceptance/bundle/variables/prepend-workspace-var/output.txt

@@ -7,7 +7,7 @@
     },
     "target": "dev",
     "terraform": {
-      "exec_path": "$TMPHOME"
+      "exec_path": "$TERRAFORM"
     }
   },
   "resources": {

acceptance/install_terraform.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""
+Script to set up terraform and databricks terraform provider in a local directory:
+
+- Download terraform.
+- Download databricks provider.
+- Write a .terraformrc config file that uses this directory.
+- The config file contains env vars that need to be set so that databricks CLI uses this terraform and provider.
+"""
+
+import os
+import platform
+import zipfile
+import argparse
+import json
+from pathlib import Path
+from urllib.request import urlretrieve
+
+os_name = platform.system().lower()
+
+arch = platform.machine().lower()
+arch = {"x86_64": "amd64"}.get(arch, arch)
+if os_name == "windows" and arch not in ("386", "amd64"):
+    # terraform 1.5.5 only has builds for these two.
+    arch = "amd64"
+
+terraform_version = "1.5.5"
+terraform_file = f"terraform_{terraform_version}_{os_name}_{arch}.zip"
+terraform_url = f"https://releases.hashicorp.com/terraform/{terraform_version}/{terraform_file}"
+terraform_binary = "terraform.exe" if os_name == "windows" else "terraform"
+
+
+def retrieve(url, path):
+    if not path.exists():
+        print(f"Downloading {url} -> {path}")
+        urlretrieve(url, path)
+
+
+def read_version(path):
+    for line in path.open():
+        if "ProviderVersion" in line:
+            # Expecting 'const ProviderVersion = "1.64.1"'
+            items = line.strip().split()
+            assert len(items) >= 3, items
+            assert items[-3:-1] == ["ProviderVersion", "="], items
+            version = items[-1].strip('"')
+            assert version, items
+            return version
+    raise SystemExit(f"Could not find ProviderVersion in {path}")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--targetdir", default="build", type=Path)
+    parser.add_argument("--provider-version")
+    args = parser.parse_args()
+    target = args.targetdir
+
+    if not args.provider_version:
+        version_file = Path(__file__).parent.parent / "bundle/internal/tf/codegen/schema/version.go"
+        assert version_file.exists(), version_file
+        terraform_provider_version = read_version(version_file)
+        print(f"Read version {terraform_provider_version} from {version_file}")
+    else:
+        terraform_provider_version = args.provider_version
+
+    terraform_provider_file = f"terraform-provider-databricks_{terraform_provider_version}_{os_name}_{arch}.zip"
+    terraform_provider_url = (
+        f"https://github.com/databricks/terraform-provider-databricks/releases/download/v{terraform_provider_version}/{terraform_provider_file}"
+    )
+
+    target.mkdir(exist_ok=True, parents=True)
+
+    zip_path = target / terraform_file
+    terraform_path = target / terraform_binary
+    terraform_provider_path = target / terraform_provider_file
+
+    retrieve(terraform_url, zip_path)
+    retrieve(terraform_provider_url, terraform_provider_path)
+
+    if not terraform_path.exists():
+        print(f"Extracting {zip_path} -> {terraform_path}")
+
+        with zipfile.ZipFile(zip_path, "r") as zip_ref:
+            zip_ref.extractall(target)
+
+        terraform_path.chmod(0o755)
+
+    tfplugins_path = target / "tfplugins"
+    provider_dir = Path(tfplugins_path / f"registry.terraform.io/databricks/databricks/{terraform_provider_version}/{os_name}_{arch}")
+    if not provider_dir.exists():
+        print(f"Extracting {terraform_provider_path} -> {provider_dir}")
+        os.makedirs(provider_dir, exist_ok=True)
+        with zipfile.ZipFile(terraform_provider_path, "r") as zip_ref:
+            zip_ref.extractall(provider_dir)
+
+        files = list(provider_dir.iterdir())
+        assert files, provider_dir
+
+        for f in files:
+            f.chmod(0o755)
+
+    terraformrc_path = target / ".terraformrc"
+    if not terraformrc_path.exists():
+        path = json.dumps(str(tfplugins_path.absolute()))
+        text = f"""# Set these env variables before running databricks cli:
+# export DATABRICKS_TF_CLI_CONFIG_FILE={terraformrc_path.absolute()}
+# export DATABRICKS_TF_EXEC_PATH={terraform_path.absolute()}
+
+provider_installation {{
+    filesystem_mirror {{
+        path = {path}
+        include = ["registry.terraform.io/databricks/databricks"]
+    }}
+}}
+"""
+        print(f"Writing {terraformrc_path}:\n{text}")
+        terraformrc_path.write_text(text)
+
+
+if __name__ == "__main__":
+    main()

acceptance/terraform/main.tf

@@ -0,0 +1,25 @@
+terraform {
+  required_providers {
+    databricks = {
+      source  = "databricks/databricks"
+      version = "1.64.1"
+    }
+  }
+
+  required_version = "= 1.5.5"
+}
+
+provider "databricks" {
+  # Optionally, specify the Databricks host and token
+  # host  = "https://<your-databricks-instance>"
+  # token = "<YOUR_PERSONAL_ACCESS_TOKEN>"
+}
+
+data "databricks_current_user" "me" {
+  # Retrieves the current user's information
+}
+
+output "username" {
+  description = "Username"
+  value       = "${data.databricks_current_user.me.user_name}"
+}

acceptance/terraform/output.txt

@@ -0,0 +1,51 @@
+
+>>> $TERRAFORM init -no-color -get=false
+
+Initializing the backend...
+
+Initializing provider plugins...
+- Finding databricks/databricks versions matching "1.64.1"...
+- Installing databricks/databricks v1.64.1...
+- Installed databricks/databricks v1.64.1 (unauthenticated)
+
+Terraform has created a lock file .terraform.lock.hcl to record the provider
+selections it made above. Include this file in your version control repository
+so that Terraform can guarantee to make the same selections by default when
+you run "terraform init" in the future.
+
+
+Warning: Incomplete lock file information for providers
+
+Due to your customized provider installation methods, Terraform was forced to
+calculate lock file checksums locally for the following providers:
+  - databricks/databricks
+
+
+To calculate additional checksums for another platform, run:
+  terraform providers lock -platform=linux_amd64
+(where linux_amd64 is the platform to generate)
+
+Terraform has been successfully initialized!
+
+You may now begin working with Terraform. Try running "terraform plan" to see
+any changes that are required for your infrastructure. All Terraform commands
+should now work.
+
+If you ever set or change modules or backend configuration for Terraform,
+rerun this command to reinitialize your working directory. If you forget, other
+commands will detect it and remind you to do so if necessary.
+
+>>> $TERRAFORM plan -no-color
+data.databricks_current_user.me: Reading...
+data.databricks_current_user.me: Read complete after 0s [id=$USER.Id]
+
+Changes to Outputs:
+  + username = "$USERNAME"
+
+You can apply this plan to save these new output values to the Terraform
+state, without changing any real infrastructure.
+
+─────────────────────────────────────────────────────────────────────────────
+
+Note: You didn't use the -out option to save this plan, so Terraform can't
+guarantee to take exactly these actions if you run "terraform apply" now.

acceptance/terraform/script

@@ -0,0 +1,14 @@
+# Want to filter out these message:
+# Mac:
+# The current .terraform.lock.hcl file only includes checksums for
+# darwin_arm64, so Terraform running on another platform will fail to install
+# these providers.
+#
+# Linux:
+# The current .terraform.lock.hcl file only includes checksums for linux_amd64,
+# so Terraform running on another platform will fail to install these
+# providers.
+
+trace $TERRAFORM init -no-color -get=false | grep -v 'includes checksums for' | grep -v 'so Terraform running on another' | grep -v 'providers\.'
+trace $TERRAFORM plan -no-color
+rm -fr .terraform.lock.hcl .terraform