Add DATABRICKS_BUNDLE_INCLUDE_PATHS to specify include paths through env vars (#591)

## Changes
* This PR adds `DATABRICKS_BUNDLE_INCLUDE_PATHS` environment variable,
so that we can specify including bundle config files, which we do not
want to commit. These could potentially be local dev overrides or
overrides by our tools - like the VS Code extension
* We always add these include paths to the "include" field. 
## Tests
* [x] Unit tests

bundle/config/mutator/process_root_includes.go

@@ -3,6 +3,7 @@ package mutator
 import (
 	"context"
 	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 
@@ -11,6 +12,17 @@ import (
 	"golang.org/x/exp/slices"
 )
 
+const ExtraIncludePathsKey string = "DATABRICKS_BUNDLE_INCLUDES"
+
+// Get extra include paths from environment variable
+func GetExtraIncludePaths() []string {
+	value, exists := os.LookupEnv(ExtraIncludePathsKey)
+	if !exists {
+		return nil
+	}
+	return strings.Split(value, string(os.PathListSeparator))
+}
+
 type processRootIncludes struct{}
 
 // ProcessRootIncludes expands the patterns in the configuration's include list
@@ -37,6 +49,18 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) error
 	// This is stored in the bundle configuration for observability.
 	var files []string
 
+	// Converts extra include paths from environment variable to relative paths
+	for _, extraIncludePath := range GetExtraIncludePaths() {
+		if filepath.IsAbs(extraIncludePath) {
+			rel, err := filepath.Rel(b.Config.Path, extraIncludePath)
+			if err != nil {
+				return fmt.Errorf("unable to include file '%s': %w", extraIncludePath, err)
+			}
+			extraIncludePath = rel
+		}
+		b.Config.Include = append(b.Config.Include, extraIncludePath)
+	}
+
 	// For each glob, find all files to load.
 	// Ordering of the list of globs is maintained in the output.
 	// For matches that appear in multiple globs, only the first is kept.

bundle/config/mutator/process_root_includes_test.go

@@ -2,7 +2,9 @@ package mutator_test
 
 import (
 	"context"
+	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime"
 	"testing"
@@ -122,3 +124,40 @@ func TestProcessRootIncludesNotExists(t *testing.T) {
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "notexist.yml defined in 'include' section does not match any files")
 }
+
+func TestProcessRootIncludesExtrasFromEnvVar(t *testing.T) {
+	rootPath := t.TempDir()
+	testYamlName := "extra_include_path.yml"
+	touch(t, rootPath, testYamlName)
+	os.Setenv(mutator.ExtraIncludePathsKey, path.Join(rootPath, testYamlName))
+	t.Cleanup(func() {
+		os.Unsetenv(mutator.ExtraIncludePathsKey)
+	})
+
+	bundle := &bundle.Bundle{
+		Config: config.Root{
+			Path: rootPath,
+		},
+	}
+
+	err := mutator.ProcessRootIncludes().Apply(context.Background(), bundle)
+	require.NoError(t, err)
+	assert.Contains(t, bundle.Config.Include, testYamlName)
+}
+
+func TestProcessRootIncludesDedupExtrasFromEnvVar(t *testing.T) {
+	rootPath := t.TempDir()
+	testYamlName := "extra_include_path.yml"
+	touch(t, rootPath, testYamlName)
+	t.Setenv(mutator.ExtraIncludePathsKey, fmt.Sprintf("%s%s%s", path.Join(rootPath, testYamlName), string(os.PathListSeparator), path.Join(rootPath, testYamlName)))
+
+	bundle := &bundle.Bundle{
+		Config: config.Root{
+			Path: rootPath,
+		},
+	}
+
+	err := mutator.ProcessRootIncludes().Apply(context.Background(), bundle)
+	require.NoError(t, err)
+	assert.Equal(t, []string{testYamlName}, bundle.Config.Include)
+}