Skeleton for configuration loading and mutation (#92)

Load a tree of configuration files anchored at `bundle.yml` into the
`config.Root` struct.

All mutations (from setting defaults to merging files) are observable
through the `mutator.Mutator` interface.

bundle/bundle.go

@@ -0,0 +1,24 @@
+package bundle
+
+import (
+	"path/filepath"
+
+	"github.com/databricks/bricks/bundle/config"
+)
+
+type Bundle struct {
+	Config config.Root
+}
+
+func Load(path string) (*Bundle, error) {
+	bundle := &Bundle{
+		Config: config.Root{
+			Path: path,
+		},
+	}
+	err := bundle.Config.Load(filepath.Join(path, config.FileName))
+	if err != nil {
+		return nil, err
+	}
+	return bundle, nil
+}

bundle/bundle_test.go

@@ -0,0 +1,21 @@
+package bundle
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoadNotExists(t *testing.T) {
+	b, err := Load("/doesntexist")
+	assert.True(t, os.IsNotExist(err))
+	assert.Nil(t, b)
+}
+
+func TestLoadExists(t *testing.T) {
+	b, err := Load("./config/tests/basic")
+	require.Nil(t, err)
+	assert.Equal(t, "basic", b.Config.Bundle.Name)
+}

bundle/config/bundle.go

@@ -0,0 +1,13 @@
+package config
+
+type Bundle struct {
+	Name string `json:"name,omitempty"`
+
+	// TODO
+	// Default cluster to run commands on (Python, Scala).
+	// DefaultCluster string `json:"default_cluster,omitempty"`
+
+	// TODO
+	// Default warehouse to run SQL on.
+	// DefaultWarehouse string `json:"default_warehouse,omitempty"`
+}

bundle/config/environment.go

@@ -0,0 +1,11 @@
+package config
+
+// Environment defines overrides for a single environment.
+// This structure is recursively merged into the root configuration.
+type Environment struct {
+	Bundle *Bundle `json:"bundle,omitempty"`
+
+	Workspace *Workspace `json:"workspace,omitempty"`
+
+	Resources *Resources `json:"resources,omitempty"`
+}

bundle/config/mutator/default_environment.go

@@ -0,0 +1,35 @@
+package mutator
+
+import (
+	"fmt"
+
+	"github.com/databricks/bricks/bundle/config"
+)
+
+type defineDefaultEnvironment struct {
+	name string
+}
+
+// DefineDefaultEnvironment adds an environment named "default"
+// to the configuration if none have been defined.
+func DefineDefaultEnvironment() Mutator {
+	return &defineDefaultEnvironment{
+		name: "default",
+	}
+}
+
+func (m *defineDefaultEnvironment) Name() string {
+	return fmt.Sprintf("DefineDefaultEnvironment(%s)", m.name)
+}
+
+func (m *defineDefaultEnvironment) Apply(root *config.Root) ([]Mutator, error) {
+	// Nothing to do if the configuration has at least 1 environment.
+	if root.Environments != nil || len(root.Environments) > 0 {
+		return nil, nil
+	}
+
+	// Define default environment.
+	root.Environments = make(map[string]*config.Environment)
+	root.Environments[m.name] = &config.Environment{}
+	return nil, nil
+}

bundle/config/mutator/default_environment_test.go

@@ -0,0 +1,31 @@
+package mutator_test
+
+import (
+	"testing"
+
+	"github.com/databricks/bricks/bundle/config"
+	"github.com/databricks/bricks/bundle/config/mutator"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDefaultEnvironment(t *testing.T) {
+	root := &config.Root{}
+	_, err := mutator.DefineDefaultEnvironment().Apply(root)
+	require.NoError(t, err)
+	env, ok := root.Environments["default"]
+	assert.True(t, ok)
+	assert.Equal(t, &config.Environment{}, env)
+}
+
+func TestDefaultEnvironmentAlreadySpecified(t *testing.T) {
+	root := &config.Root{
+		Environments: map[string]*config.Environment{
+			"development": {},
+		},
+	}
+	_, err := mutator.DefineDefaultEnvironment().Apply(root)
+	require.NoError(t, err)
+	_, ok := root.Environments["default"]
+	assert.False(t, ok)
+}

bundle/config/mutator/default_include.go

@@ -0,0 +1,34 @@
+package mutator
+
+import (
+	"github.com/databricks/bricks/bundle/config"
+	"golang.org/x/exp/slices"
+)
+
+type defineDefaultInclude struct {
+	include []string
+}
+
+// DefineDefaultInclude sets the list of includes to a default if it hasn't been set.
+func DefineDefaultInclude() Mutator {
+	return &defineDefaultInclude{
+		// When we support globstar we can collapse below into a single line.
+		include: []string{
+			// Load YAML files in the same directory.
+			"*.yml",
+			// Load YAML files in subdirectories.
+			"*/*.yml",
+		},
+	}
+}
+
+func (m *defineDefaultInclude) Name() string {
+	return "DefineDefaultInclude"
+}
+
+func (m *defineDefaultInclude) Apply(root *config.Root) ([]Mutator, error) {
+	if len(root.Include) == 0 {
+		root.Include = slices.Clone(m.include)
+	}
+	return nil, nil
+}

bundle/config/mutator/default_include_test.go

@@ -0,0 +1,17 @@
+package mutator_test
+
+import (
+	"testing"
+
+	"github.com/databricks/bricks/bundle/config"
+	"github.com/databricks/bricks/bundle/config/mutator"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDefaultInclude(t *testing.T) {
+	root := &config.Root{}
+	_, err := mutator.DefineDefaultInclude().Apply(root)
+	require.NoError(t, err)
+	assert.Equal(t, []string{"*.yml", "*/*.yml"}, root.Include)
+}

bundle/config/mutator/mutator.go

@@ -0,0 +1,46 @@
+package mutator
+
+import "github.com/databricks/bricks/bundle/config"
+
+// Mutator is the interface types that mutate the bundle configuration.
+// This makes every mutation observable and debuggable.
+type Mutator interface {
+	// Name returns the mutators name.
+	Name() string
+
+	// Apply mutates the specified configuration object.
+	// It optionally returns a list of mutators to invoke immediately after this mutator.
+	// This is used when processing all configuration files in the tree; each file gets
+	// its own mutator instance.
+	Apply(*config.Root) ([]Mutator, error)
+}
+
+func DefaultMutators() []Mutator {
+	return []Mutator{
+		DefineDefaultInclude(),
+		ProcessRootIncludes(),
+		DefineDefaultEnvironment(),
+	}
+}
+
+func DefaultMutatorsForEnvironment(env string) []Mutator {
+	return append(DefaultMutators(), SelectEnvironment(env))
+}
+
+func Apply(root *config.Root, ms []Mutator) error {
+	if len(ms) == 0 {
+		return nil
+	}
+	for _, m := range ms {
+		ms_, err := m.Apply(root)
+		if err != nil {
+			return err
+		}
+		// Apply recursively.
+		err = Apply(root, ms_)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

bundle/config/mutator/process_include.go

@@ -0,0 +1,32 @@
+package mutator
+
+import (
+	"fmt"
+
+	"github.com/databricks/bricks/bundle/config"
+)
+
+type processInclude struct {
+	fullPath string
+	relPath  string
+}
+
+// ProcessInclude loads the configuration at [fullPath] and merges it into the configuration.
+func ProcessInclude(fullPath, relPath string) Mutator {
+	return &processInclude{
+		fullPath: fullPath,
+		relPath:  relPath,
+	}
+}
+
+func (m *processInclude) Name() string {
+	return fmt.Sprintf("ProcessInclude(%s)", m.relPath)
+}
+
+func (m *processInclude) Apply(root *config.Root) ([]Mutator, error) {
+	this, err := config.Load(m.fullPath)
+	if err != nil {
+		return nil, err
+	}
+	return nil, root.Merge(this)
+}

bundle/config/mutator/process_include_test.go

@@ -0,0 +1,34 @@
+package mutator_test
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/bricks/bundle/config"
+	"github.com/databricks/bricks/bundle/config/mutator"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestProcessInclude(t *testing.T) {
+	root := &config.Root{
+		Path: t.TempDir(),
+		Workspace: config.Workspace{
+			Host: "foo",
+		},
+	}
+
+	relPath := "./file.yml"
+	fullPath := filepath.Join(root.Path, relPath)
+	f, err := os.Create(fullPath)
+	require.NoError(t, err)
+	fmt.Fprint(f, "workspace:\n  host: bar\n")
+	f.Close()
+
+	assert.Equal(t, "foo", root.Workspace.Host)
+	_, err = mutator.ProcessInclude(fullPath, relPath).Apply(root)
+	require.NoError(t, err)
+	assert.Equal(t, "bar", root.Workspace.Host)
+}

bundle/config/mutator/process_root_includes.go

@@ -0,0 +1,68 @@
+package mutator
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/databricks/bricks/bundle/config"
+	"golang.org/x/exp/slices"
+)
+
+type processRootIncludes struct{}
+
+// ProcessRootIncludes expands the patterns in the configuration's include list
+// into a list of mutators for each matching file.
+func ProcessRootIncludes() Mutator {
+	return &processRootIncludes{}
+}
+
+func (m *processRootIncludes) Name() string {
+	return "ProcessRootIncludes"
+}
+
+func (m *processRootIncludes) Apply(root *config.Root) ([]Mutator, error) {
+	var out []Mutator
+
+	// Map with files we've already seen to avoid loading them twice.
+	var seen = map[string]bool{
+		config.FileName: true,
+	}
+
+	// For each glob, find all files to load.
+	// Ordering of the list of globs is maintained in the output.
+	// For matches that appear in multiple globs, only the first is kept.
+	for _, entry := range root.Include {
+		// Include paths must be relative.
+		if filepath.IsAbs(entry) {
+			return nil, fmt.Errorf("%s: includes must be relative paths", entry)
+		}
+
+		// Anchor includes to the bundle root path.
+		matches, err := filepath.Glob(filepath.Join(root.Path, entry))
+		if err != nil {
+			return nil, err
+		}
+
+		// Filter matches to ones we haven't seen yet.
+		var includes []string
+		for _, match := range matches {
+			rel, err := filepath.Rel(root.Path, match)
+			if err != nil {
+				return nil, err
+			}
+			if _, ok := seen[rel]; ok {
+				continue
+			}
+			seen[rel] = true
+			includes = append(includes, rel)
+		}
+
+		// Add matches to list of mutators to return.
+		slices.Sort(includes)
+		for _, include := range includes {
+			out = append(out, ProcessInclude(filepath.Join(root.Path, include), include))
+		}
+	}
+
+	return out, nil
+}

bundle/config/mutator/process_root_includes_test.go

@@ -0,0 +1,102 @@
+package mutator_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/bricks/bundle/config"
+	"github.com/databricks/bricks/bundle/config/mutator"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func touch(t *testing.T, path, file string) {
+	f, err := os.Create(filepath.Join(path, file))
+	require.NoError(t, err)
+	f.Close()
+}
+
+func TestProcessRootIncludesEmpty(t *testing.T) {
+	root := &config.Root{Path: "."}
+	_, err := mutator.ProcessRootIncludes().Apply(root)
+	require.NoError(t, err)
+}
+
+func TestProcessRootIncludesAbs(t *testing.T) {
+	root := &config.Root{
+		Path: ".",
+		Include: []string{
+			"/tmp/*.yml",
+		},
+	}
+	_, err := mutator.ProcessRootIncludes().Apply(root)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "must be relative paths")
+}
+
+func TestProcessRootIncludesSingleGlob(t *testing.T) {
+	root := &config.Root{
+		Path: t.TempDir(),
+		Include: []string{
+			"*.yml",
+		},
+	}
+
+	touch(t, root.Path, "bundle.yml")
+	touch(t, root.Path, "a.yml")
+	touch(t, root.Path, "b.yml")
+
+	ms, err := mutator.ProcessRootIncludes().Apply(root)
+	require.NoError(t, err)
+
+	var names []string
+	for _, m := range ms {
+		names = append(names, m.Name())
+	}
+
+	assert.NotContains(t, names, "ProcessInclude(bundle.yml)")
+	assert.Contains(t, names, "ProcessInclude(a.yml)")
+	assert.Contains(t, names, "ProcessInclude(b.yml)")
+}
+
+func TestProcessRootIncludesMultiGlob(t *testing.T) {
+	root := &config.Root{
+		Path: t.TempDir(),
+		Include: []string{
+			"a*.yml",
+			"b*.yml",
+		},
+	}
+
+	touch(t, root.Path, "a1.yml")
+	touch(t, root.Path, "b1.yml")
+
+	ms, err := mutator.ProcessRootIncludes().Apply(root)
+	require.NoError(t, err)
+
+	var names []string
+	for _, m := range ms {
+		names = append(names, m.Name())
+	}
+
+	assert.Contains(t, names, "ProcessInclude(a1.yml)")
+	assert.Contains(t, names, "ProcessInclude(b1.yml)")
+}
+
+func TestProcessRootIncludesRemoveDups(t *testing.T) {
+	root := &config.Root{
+		Path: t.TempDir(),
+		Include: []string{
+			"*.yml",
+			"*.yml",
+		},
+	}
+
+	touch(t, root.Path, "a.yml")
+
+	ms, err := mutator.ProcessRootIncludes().Apply(root)
+	require.NoError(t, err)
+	assert.Len(t, ms, 1)
+	assert.Equal(t, "ProcessInclude(a.yml)", ms[0].Name())
+}

bundle/config/mutator/select_environment.go

@@ -0,0 +1,44 @@
+package mutator
+
+import (
+	"fmt"
+
+	"github.com/databricks/bricks/bundle/config"
+)
+
+type selectEnvironment struct {
+	name string
+}
+
+// SelectEnvironment merges the specified environment into the root configuration.
+func SelectEnvironment(name string) Mutator {
+	return &selectEnvironment{
+		name: name,
+	}
+}
+
+func (m *selectEnvironment) Name() string {
+	return fmt.Sprintf("SelectEnvironment(%s)", m.name)
+}
+
+func (m *selectEnvironment) Apply(root *config.Root) ([]Mutator, error) {
+	if root.Environments == nil {
+		return nil, fmt.Errorf("no environments defined")
+	}
+
+	// Get specified environment
+	env, ok := root.Environments[m.name]
+	if !ok {
+		return nil, fmt.Errorf("%s: no such environment", m.name)
+	}
+
+	// Merge specified environment into root configuration structure.
+	err := root.MergeEnvironment(env)
+	if err != nil {
+		return nil, err
+	}
+
+	// Clear environments after loading.
+	root.Environments = nil
+	return nil, nil
+}

bundle/config/mutator/select_environment_test.go

@@ -0,0 +1,38 @@
+package mutator_test
+
+import (
+	"testing"
+
+	"github.com/databricks/bricks/bundle/config"
+	"github.com/databricks/bricks/bundle/config/mutator"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSelectEnvironment(t *testing.T) {
+	root := &config.Root{
+		Workspace: config.Workspace{
+			Host: "foo",
+		},
+		Environments: map[string]*config.Environment{
+			"default": {
+				Workspace: &config.Workspace{
+					Host: "bar",
+				},
+			},
+		},
+	}
+	_, err := mutator.SelectEnvironment("default").Apply(root)
+	require.NoError(t, err)
+	assert.Equal(t, "bar", root.Workspace.Host)
+}
+
+func TestSelectEnvironmentNotFound(t *testing.T) {
+	root := &config.Root{
+		Environments: map[string]*config.Environment{
+			"default": {},
+		},
+	}
+	_, err := mutator.SelectEnvironment("doesnt-exist").Apply(root)
+	require.Error(t, err, "no environments defined")
+}

bundle/config/resources.go

@@ -0,0 +1,9 @@
+package config
+
+import "github.com/databricks/bricks/bundle/config/resources"
+
+// Resources defines Databricks resources associated with the bundle.
+type Resources struct {
+	Jobs      map[string]resources.Job      `json:"jobs,omitempty"`
+	Pipelines map[string]resources.Pipeline `json:"pipelines,omitempty"`
+}

bundle/config/resources/job.go

@@ -0,0 +1,5 @@
+package resources
+
+type Job struct {
+	ID string `json:"id,omitempty"`
+}

bundle/config/resources/pipeline.go

@@ -0,0 +1,5 @@
+package resources
+
+type Pipeline struct {
+	ID string `json:"id,omitempty"`
+}

bundle/config/root.go

@@ -0,0 +1,109 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/ghodss/yaml"
+	"github.com/imdario/mergo"
+)
+
+// FileName is the name of bundle configuration file.
+const FileName = "bundle.yml"
+
+type Root struct {
+	// Path contains the directory path to the root of the bundle.
+	// It is set when loading `bundle.yml`.
+	Path string `json:"-"`
+
+	// Bundle contains details about this bundle, such as its name,
+	// version of the spec (TODO), default cluster, default warehouse, etc.
+	Bundle Bundle `json:"bundle"`
+
+	// Include specifies a list of patterns of file names to load and
+	// merge into the this configuration. If not set in `bundle.yml`,
+	// it defaults to loading `*.yml` and `*/*.yml`.
+	//
+	// Also see [mutator.DefineDefaultInclude].
+	//
+	Include []string `json:"include,omitempty"`
+
+	// Workspace contains details about the workspace to connect to
+	// and paths in the workspace tree to use for this bundle.
+	Workspace Workspace `json:"workspace"`
+
+	// Resources contains a description of all Databricks resources
+	// to deploy in this bundle (e.g. jobs, pipelines, etc.).
+	Resources Resources `json:"resources"`
+
+	// Environments can be used to differentiate settings and resources between
+	// bundle deployment environments (e.g. development, staging, production).
+	// If not specified, the code below initializes this field with a
+	// single default-initialized environment called "default".
+	Environments map[string]*Environment `json:"environments,omitempty"`
+}
+
+func Load(path string) (*Root, error) {
+	var r Root
+
+	stat, err := os.Stat(path)
+	if err != nil {
+		return nil, err
+	}
+
+	// If we were given a directory, assume this is the bundle root.
+	if stat.IsDir() {
+		r.Path = path
+		path = filepath.Join(path, FileName)
+	}
+
+	if err := r.Load(path); err != nil {
+		return nil, err
+	}
+
+	return &r, nil
+}
+
+func (r *Root) Load(file string) error {
+	raw, err := os.ReadFile(file)
+	if err != nil {
+		return err
+	}
+	err = yaml.Unmarshal(raw, r)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (r *Root) Merge(other *Root) error {
+	// TODO: define and test semantics for merging.
+	return mergo.MergeWithOverwrite(r, other)
+}
+
+func (r *Root) MergeEnvironment(env *Environment) error {
+	var err error
+
+	if env.Bundle != nil {
+		err = mergo.MergeWithOverwrite(&r.Bundle, env.Bundle)
+		if err != nil {
+			return err
+		}
+	}
+
+	if env.Workspace != nil {
+		err = mergo.MergeWithOverwrite(&r.Workspace, env.Workspace)
+		if err != nil {
+			return err
+		}
+	}
+
+	if env.Resources != nil {
+		err = mergo.MergeWithOverwrite(&r.Resources, env.Resources)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

bundle/config/root_test.go

@@ -0,0 +1,72 @@
+package config
+
+import (
+	"encoding/json"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRootMarshalUnmarshal(t *testing.T) {
+	// Marshal empty
+	buf, err := json.Marshal(&Root{})
+	require.NoError(t, err)
+
+	// Unmarshal empty
+	var root Root
+	err = json.Unmarshal(buf, &root)
+	require.NoError(t, err)
+
+	// Compare
+	assert.True(t, reflect.DeepEqual(Root{}, root))
+}
+
+func TestRootLoad(t *testing.T) {
+	root := &Root{}
+	err := root.Load("./tests/basic/bundle.yml")
+	require.NoError(t, err)
+	assert.Equal(t, "basic", root.Bundle.Name)
+}
+
+func TestRootMergeStruct(t *testing.T) {
+	root := &Root{
+		Workspace: Workspace{
+			Host:    "foo",
+			Profile: "profile",
+		},
+	}
+	other := &Root{
+		Workspace: Workspace{
+			Host: "bar",
+		},
+	}
+	assert.NoError(t, root.Merge(other))
+	assert.Equal(t, "bar", root.Workspace.Host)
+	assert.Equal(t, "profile", root.Workspace.Profile)
+}
+
+func TestRootMergeMap(t *testing.T) {
+	root := &Root{
+		Environments: map[string]*Environment{
+			"development": {
+				Workspace: &Workspace{
+					Host:    "foo",
+					Profile: "profile",
+				},
+			},
+		},
+	}
+	other := &Root{
+		Environments: map[string]*Environment{
+			"development": {
+				Workspace: &Workspace{
+					Host: "bar",
+				},
+			},
+		},
+	}
+	assert.NoError(t, root.Merge(other))
+	assert.Equal(t, &Workspace{Host: "bar", Profile: "profile"}, root.Environments["development"].Workspace)
+}

bundle/config/tests/README.md

@@ -0,0 +1,4 @@
+# Bundle configuration tests
+
+Every test here uses an example bundle configuration.
+Each bundle configuration is located in a dedicated subdirectory.

bundle/config/tests/basic/bundle.yml

@@ -0,0 +1,2 @@
+bundle:
+  name: basic

bundle/config/tests/basic_test.go

@@ -0,0 +1,12 @@
+package config_tests
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBasic(t *testing.T) {
+	root := load(t, "./basic")
+	assert.Equal(t, "basic", root.Bundle.Name)
+}

bundle/config/tests/environment_overrides/bundle.yml

@@ -0,0 +1,14 @@
+bundle:
+  name: environment_overrides
+
+workspace:
+  host: https://acme.cloud.databricks.com/
+
+environments:
+  development:
+    workspace:
+      host: https://development.acme.cloud.databricks.com/
+
+  staging:
+    workspace:
+      host: https://staging.acme.cloud.databricks.com/

bundle/config/tests/environment_overrides_test.go

@@ -0,0 +1,14 @@
+package config_tests
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEnvironmentOverridesDev(t *testing.T) {
+	development := loadEnvironment(t, "./environment_overrides", "development")
+	assert.Equal(t, "https://development.acme.cloud.databricks.com/", development.Workspace.Host)
+	staging := loadEnvironment(t, "./environment_overrides", "staging")
+	assert.Equal(t, "https://staging.acme.cloud.databricks.com/", staging.Workspace.Host)
+}

bundle/config/tests/include_default/bundle.yml

@@ -0,0 +1,2 @@
+bundle:
+  name: include_default

bundle/config/tests/include_default/my_first_job/resource.yml

@@ -0,0 +1,4 @@
+resources:
+  jobs:
+    my_first_job:
+      id: 1

bundle/config/tests/include_default/my_second_job/resource.yml

@@ -0,0 +1,4 @@
+resources:
+  jobs:
+    my_second_job:
+      id: 2

bundle/config/tests/include_default_test.go

@@ -0,0 +1,20 @@
+package config_tests
+
+import (
+	"sort"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/exp/maps"
+)
+
+func TestIncludeDefault(t *testing.T) {
+	root := load(t, "./include_default")
+
+	// Test that both jobs were loaded.
+	keys := maps.Keys(root.Resources.Jobs)
+	sort.Strings(keys)
+	assert.Equal(t, []string{"my_first_job", "my_second_job"}, keys)
+	assert.Equal(t, "1", root.Resources.Jobs["my_first_job"].ID)
+	assert.Equal(t, "2", root.Resources.Jobs["my_second_job"].ID)
+}

bundle/config/tests/include_override/bundle.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: include_override
+
+# Setting this explicitly means default globs are not processed.
+# As a result, ./this_file_isnt_included.yml isn't included.
+include:
+  - doesnt-exist/*.yml

bundle/config/tests/include_override/this_file_isnt_included.yml

@@ -0,0 +1,4 @@
+resources:
+  jobs:
+    this_job_isnt_defined:
+      id: 1

bundle/config/tests/include_override_test.go

@@ -0,0 +1,12 @@
+package config_tests
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIncludeOverride(t *testing.T) {
+	root := load(t, "./include_override")
+	assert.Empty(t, root.Resources.Jobs)
+}

bundle/config/tests/loader.go

@@ -0,0 +1,24 @@
+package config_tests
+
+import (
+	"testing"
+
+	"github.com/databricks/bricks/bundle/config"
+	"github.com/databricks/bricks/bundle/config/mutator"
+	"github.com/stretchr/testify/require"
+)
+
+func load(t *testing.T, path string) *config.Root {
+	root, err := config.Load(path)
+	require.NoError(t, err)
+	err = mutator.Apply(root, mutator.DefaultMutators())
+	require.NoError(t, err)
+	return root
+}
+
+func loadEnvironment(t *testing.T, path, env string) *config.Root {
+	root := load(t, path)
+	err := mutator.Apply(root, []mutator.Mutator{mutator.SelectEnvironment(env)})
+	require.NoError(t, err)
+	return root
+}

bundle/config/workspace.go

@@ -0,0 +1,8 @@
+package config
+
+// Workspace defines configurables at the workspace level.
+type Workspace struct {
+	// TODO: Add all unified authentication configurables.
+	Host    string `json:"host,omitempty"`
+	Profile string `json:"profile,omitempty"`
+}

bundle/root.go

@@ -0,0 +1,38 @@
+package bundle
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/databricks/bricks/bundle/config"
+	"github.com/databricks/bricks/folders"
+)
+
+const envBundleRoot = "BUNDLE_ROOT"
+
+// getRoot returns the bundle root.
+// If the `BUNDLE_ROOT` environment variable is set, we assume its value
+// to be a valid bundle root. Otherwise we try to find it by traversing
+// the path and looking for a project configuration file.
+func getRoot() (string, error) {
+	path, ok := os.LookupEnv(envBundleRoot)
+	if ok {
+		stat, err := os.Stat(path)
+		if err == nil && !stat.IsDir() {
+			err = fmt.Errorf("not a directory")
+		}
+		if err != nil {
+			return "", fmt.Errorf(`invalid bundle root %s="%s": %w`, envBundleRoot, path, err)
+		}
+		return path, nil
+	}
+	wd, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+	path, err = folders.FindDirWithLeaf(wd, config.FileName)
+	if err != nil {
+		return "", fmt.Errorf(`unable to locate bundle root`)
+	}
+	return path, nil
+}

bundle/root_test.go

@@ -0,0 +1,104 @@
+package bundle
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/bricks/bundle/config"
+	"github.com/stretchr/testify/require"
+)
+
+// Changes into specified directory for the duration of the test.
+// Returns the current working directory.
+func chdir(t *testing.T, dir string) string {
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+
+	abs, err := filepath.Abs(dir)
+	require.NoError(t, err)
+
+	err = os.Chdir(abs)
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		err := os.Chdir(wd)
+		require.NoError(t, err)
+	})
+
+	return wd
+}
+
+func TestRootFromEnv(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv(envBundleRoot, dir)
+
+	// It should pull the root from the environment variable.
+	root, err := getRoot()
+	require.NoError(t, err)
+	require.Equal(t, root, dir)
+}
+
+func TestRootFromEnvDoesntExist(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv(envBundleRoot, filepath.Join(dir, "doesntexist"))
+
+	// It should pull the root from the environment variable.
+	_, err := getRoot()
+	require.Errorf(t, err, "invalid bundle root")
+}
+
+func TestRootFromEnvIsFile(t *testing.T) {
+	dir := t.TempDir()
+	f, err := os.Create(filepath.Join(dir, "invalid"))
+	require.NoError(t, err)
+	f.Close()
+	t.Setenv(envBundleRoot, f.Name())
+
+	// It should pull the root from the environment variable.
+	_, err = getRoot()
+	require.Errorf(t, err, "invalid bundle root")
+}
+
+func TestRootIfEnvIsEmpty(t *testing.T) {
+	dir := ""
+	t.Setenv(envBundleRoot, dir)
+
+	// It should pull the root from the environment variable.
+	_, err := getRoot()
+	require.Errorf(t, err, "invalid bundle root")
+}
+
+func TestRootLookup(t *testing.T) {
+	// Have to set then unset to allow the testing package to revert it to its original value.
+	t.Setenv(envBundleRoot, "")
+	os.Unsetenv(envBundleRoot)
+
+	chdir(t, t.TempDir())
+
+	// Create bundle.yml file.
+	f, err := os.Create(config.FileName)
+	require.NoError(t, err)
+	defer f.Close()
+
+	// Create directory tree.
+	err = os.MkdirAll("./a/b/c", 0755)
+	require.NoError(t, err)
+
+	// It should find the project root from $PWD.
+	wd := chdir(t, "./a/b/c")
+	root, err := getRoot()
+	require.NoError(t, err)
+	require.Equal(t, wd, root)
+}
+
+func TestRootLookupError(t *testing.T) {
+	// Have to set then unset to allow the testing package to revert it to its original value.
+	t.Setenv(envBundleRoot, "")
+	os.Unsetenv(envBundleRoot)
+
+	// It can't find a project root from a temporary directory.
+	_ = chdir(t, t.TempDir())
+	_, err := getRoot()
+	require.ErrorContains(t, err, "unable to locate bundle root")
+}

go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
+	github.com/imdario/mergo v0.3.13
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

go.sum

@@ -59,6 +59,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs=
 github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
 github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ=
+github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
+github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
@@ -182,6 +184,7 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=