Add DATABRICKS_BUNDLE_TMP env variable (#462)

## Changes
Add DATABRICKS_BUNDLE_TMP env variable. It allows using a temporary
directory instead of writing to `$CWD/.databricks/bundle`

## Tests
I added unit tests

---------

Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>

bundle/bundle.go

@@ -91,8 +91,6 @@ func (b *Bundle) WorkspaceClient() *databricks.WorkspaceClient {
 	return b.client
 }
 
-var cacheDirName = filepath.Join(".databricks", "bundle")
-
 // CacheDir returns directory to use for temporary files for this bundle.
 // Scoped to the bundle's environment.
 func (b *Bundle) CacheDir(paths ...string) (string, error) {
@@ -100,11 +98,20 @@ func (b *Bundle) CacheDir(paths ...string) (string, error) {
 		panic("environment not set")
 	}
 
-	// Fixed components of the result path.
-	parts := []string{
+	cacheDirName, exists := os.LookupEnv("DATABRICKS_BUNDLE_TMP")
+
+	if !exists || cacheDirName == "" {
+		cacheDirName = filepath.Join(
 			// Anchor at bundle root directory.
 			b.Config.Path,
 			// Static cache directory.
+			".databricks",
+			"bundle",
+		)
+	}
+
+	// Fixed components of the result path.
+	parts := []string{
 		cacheDirName,
 		// Scope with environment name.
 		b.Config.Bundle.Environment,

bundle/bundle_test.go

@@ -3,7 +3,6 @@ package bundle
 import (
 	"os"
 	"path/filepath"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -35,9 +34,38 @@ func TestBundleCacheDir(t *testing.T) {
 	// This is otherwise done by [mutators.SelectEnvironment].
 	bundle.Config.Bundle.Environment = "default"
 
+	// unset env variable in case it's set
+	t.Setenv("DATABRICKS_BUNDLE_TMP", "")
+
 	cacheDir, err := bundle.CacheDir()
+
+	// format is <CWD>/.databricks/bundle/<environment>
 	assert.NoError(t, err)
-	assert.True(t, strings.HasPrefix(cacheDir, projectDir))
+	assert.Equal(t, filepath.Join(projectDir, ".databricks", "bundle", "default"), cacheDir)
+}
+
+func TestBundleCacheDirOverride(t *testing.T) {
+	projectDir := t.TempDir()
+	bundleTmpDir := t.TempDir()
+	f1, err := os.Create(filepath.Join(projectDir, "bundle.yml"))
+	require.NoError(t, err)
+	f1.Close()
+
+	bundle, err := Load(projectDir)
+	require.NoError(t, err)
+
+	// Artificially set environment.
+	// This is otherwise done by [mutators.SelectEnvironment].
+	bundle.Config.Bundle.Environment = "default"
+
+	// now we expect to use 'bundleTmpDir' instead of CWD/.databricks/bundle
+	t.Setenv("DATABRICKS_BUNDLE_TMP", bundleTmpDir)
+
+	cacheDir, err := bundle.CacheDir()
+
+	// format is <DATABRICKS_BUNDLE_TMP>/<environment>
+	assert.NoError(t, err)
+	assert.Equal(t, filepath.Join(bundleTmpDir, "default"), cacheDir)
 }
 
 func TestBundleMustLoadSuccess(t *testing.T) {