diff --git a/bundle/bundle.go b/bundle/bundle.go index 076294d7..ab85a057 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -1,12 +1,10 @@ package bundle import ( - "context" "path/filepath" "sync" "github.com/databricks/bricks/bundle/config" - "github.com/databricks/bricks/bundle/config/mutator" "github.com/databricks/databricks-sdk-go" ) @@ -19,10 +17,6 @@ type Bundle struct { client *databricks.WorkspaceClient } -func (b *Bundle) MutateForEnvironment(env string) error { - return mutator.Apply(&b.Config, mutator.DefaultMutatorsForEnvironment(env)) -} - func Load(path string) (*Bundle, error) { bundle := &Bundle{ Config: config.Root{ @@ -45,20 +39,6 @@ func LoadFromRoot() (*Bundle, error) { return Load(root) } -func ConfigureForEnvironment(ctx context.Context, env string) (context.Context, error) { - b, err := LoadFromRoot() - if err != nil { - return nil, err - } - - err = b.MutateForEnvironment(env) - if err != nil { - return nil, err - } - - return Context(ctx, b), nil -} - func (b *Bundle) WorkspaceClient() *databricks.WorkspaceClient { b.clientOnce.Do(func() { var err error diff --git a/bundle/bundle_test.go b/bundle/bundle_test.go index ef4df7f9..9196563d 100644 --- a/bundle/bundle_test.go +++ b/bundle/bundle_test.go @@ -15,7 +15,7 @@ func TestLoadNotExists(t *testing.T) { } func TestLoadExists(t *testing.T) { - b, err := Load("./config/tests/basic") + b, err := Load("./tests/basic") require.Nil(t, err) assert.Equal(t, "basic", b.Config.Bundle.Name) } diff --git a/bundle/config/mutator/default_environment.go b/bundle/config/mutator/default_environment.go index aa79e600..6e7ecdeb 100644 --- a/bundle/config/mutator/default_environment.go +++ b/bundle/config/mutator/default_environment.go @@ -1,8 +1,10 @@ package mutator import ( + "context" "fmt" + "github.com/databricks/bricks/bundle" "github.com/databricks/bricks/bundle/config" ) @@ -12,7 +14,7 @@ type defineDefaultEnvironment struct { // DefineDefaultEnvironment adds an environment named "default" // to the configuration if none have been defined. -func DefineDefaultEnvironment() Mutator { +func DefineDefaultEnvironment() bundle.Mutator { return &defineDefaultEnvironment{ name: "default", } @@ -22,14 +24,14 @@ func (m *defineDefaultEnvironment) Name() string { return fmt.Sprintf("DefineDefaultEnvironment(%s)", m.name) } -func (m *defineDefaultEnvironment) Apply(root *config.Root) ([]Mutator, error) { +func (m *defineDefaultEnvironment) Apply(_ context.Context, b *bundle.Bundle) ([]bundle.Mutator, error) { // Nothing to do if the configuration has at least 1 environment. - if root.Environments != nil || len(root.Environments) > 0 { + if b.Config.Environments != nil || len(b.Config.Environments) > 0 { return nil, nil } // Define default environment. - root.Environments = make(map[string]*config.Environment) - root.Environments[m.name] = &config.Environment{} + b.Config.Environments = make(map[string]*config.Environment) + b.Config.Environments[m.name] = &config.Environment{} return nil, nil } diff --git a/bundle/config/mutator/default_environment_test.go b/bundle/config/mutator/default_environment_test.go index 8b250d2c..25a1c196 100644 --- a/bundle/config/mutator/default_environment_test.go +++ b/bundle/config/mutator/default_environment_test.go @@ -1,8 +1,10 @@ package mutator_test import ( + "context" "testing" + "github.com/databricks/bricks/bundle" "github.com/databricks/bricks/bundle/config" "github.com/databricks/bricks/bundle/config/mutator" "github.com/stretchr/testify/assert" @@ -10,22 +12,24 @@ import ( ) func TestDefaultEnvironment(t *testing.T) { - root := &config.Root{} - _, err := mutator.DefineDefaultEnvironment().Apply(root) + bundle := &bundle.Bundle{} + _, err := mutator.DefineDefaultEnvironment().Apply(context.Background(), bundle) require.NoError(t, err) - env, ok := root.Environments["default"] + env, ok := bundle.Config.Environments["default"] assert.True(t, ok) assert.Equal(t, &config.Environment{}, env) } func TestDefaultEnvironmentAlreadySpecified(t *testing.T) { - root := &config.Root{ - Environments: map[string]*config.Environment{ - "development": {}, + bundle := &bundle.Bundle{ + Config: config.Root{ + Environments: map[string]*config.Environment{ + "development": {}, + }, }, } - _, err := mutator.DefineDefaultEnvironment().Apply(root) + _, err := mutator.DefineDefaultEnvironment().Apply(context.Background(), bundle) require.NoError(t, err) - _, ok := root.Environments["default"] + _, ok := bundle.Config.Environments["default"] assert.False(t, ok) } diff --git a/bundle/config/mutator/default_include.go b/bundle/config/mutator/default_include.go index 8b9a68d2..2a8dda52 100644 --- a/bundle/config/mutator/default_include.go +++ b/bundle/config/mutator/default_include.go @@ -1,7 +1,9 @@ package mutator import ( - "github.com/databricks/bricks/bundle/config" + "context" + + "github.com/databricks/bricks/bundle" "golang.org/x/exp/slices" ) @@ -10,7 +12,7 @@ type defineDefaultInclude struct { } // DefineDefaultInclude sets the list of includes to a default if it hasn't been set. -func DefineDefaultInclude() Mutator { +func DefineDefaultInclude() bundle.Mutator { return &defineDefaultInclude{ // When we support globstar we can collapse below into a single line. include: []string{ @@ -26,9 +28,9 @@ func (m *defineDefaultInclude) Name() string { return "DefineDefaultInclude" } -func (m *defineDefaultInclude) Apply(root *config.Root) ([]Mutator, error) { - if len(root.Include) == 0 { - root.Include = slices.Clone(m.include) +func (m *defineDefaultInclude) Apply(_ context.Context, b *bundle.Bundle) ([]bundle.Mutator, error) { + if len(b.Config.Include) == 0 { + b.Config.Include = slices.Clone(m.include) } return nil, nil } diff --git a/bundle/config/mutator/default_include_test.go b/bundle/config/mutator/default_include_test.go index 1ffa1766..11500529 100644 --- a/bundle/config/mutator/default_include_test.go +++ b/bundle/config/mutator/default_include_test.go @@ -1,17 +1,18 @@ package mutator_test import ( + "context" "testing" - "github.com/databricks/bricks/bundle/config" + "github.com/databricks/bricks/bundle" "github.com/databricks/bricks/bundle/config/mutator" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestDefaultInclude(t *testing.T) { - root := &config.Root{} - _, err := mutator.DefineDefaultInclude().Apply(root) + bundle := &bundle.Bundle{} + _, err := mutator.DefineDefaultInclude().Apply(context.Background(), bundle) require.NoError(t, err) - assert.Equal(t, []string{"*.yml", "*/*.yml"}, root.Include) + assert.Equal(t, []string{"*.yml", "*/*.yml"}, bundle.Config.Include) } diff --git a/bundle/config/mutator/mutator.go b/bundle/config/mutator/mutator.go index 4e42e908..99e1bffe 100644 --- a/bundle/config/mutator/mutator.go +++ b/bundle/config/mutator/mutator.go @@ -1,46 +1,17 @@ package mutator -import "github.com/databricks/bricks/bundle/config" +import ( + "github.com/databricks/bricks/bundle" +) -// Mutator is the interface types that mutate the bundle configuration. -// This makes every mutation observable and debuggable. -type Mutator interface { - // Name returns the mutators name. - Name() string - - // Apply mutates the specified configuration object. - // It optionally returns a list of mutators to invoke immediately after this mutator. - // This is used when processing all configuration files in the tree; each file gets - // its own mutator instance. - Apply(*config.Root) ([]Mutator, error) -} - -func DefaultMutators() []Mutator { - return []Mutator{ +func DefaultMutators() []bundle.Mutator { + return []bundle.Mutator{ DefineDefaultInclude(), ProcessRootIncludes(), DefineDefaultEnvironment(), } } -func DefaultMutatorsForEnvironment(env string) []Mutator { +func DefaultMutatorsForEnvironment(env string) []bundle.Mutator { return append(DefaultMutators(), SelectEnvironment(env)) } - -func Apply(root *config.Root, ms []Mutator) error { - if len(ms) == 0 { - return nil - } - for _, m := range ms { - ms_, err := m.Apply(root) - if err != nil { - return err - } - // Apply recursively. - err = Apply(root, ms_) - if err != nil { - return err - } - } - return nil -} diff --git a/bundle/config/mutator/process_include.go b/bundle/config/mutator/process_include.go index 6b0f34e3..91e7ac9a 100644 --- a/bundle/config/mutator/process_include.go +++ b/bundle/config/mutator/process_include.go @@ -1,8 +1,10 @@ package mutator import ( + "context" "fmt" + "github.com/databricks/bricks/bundle" "github.com/databricks/bricks/bundle/config" ) @@ -12,7 +14,7 @@ type processInclude struct { } // ProcessInclude loads the configuration at [fullPath] and merges it into the configuration. -func ProcessInclude(fullPath, relPath string) Mutator { +func ProcessInclude(fullPath, relPath string) bundle.Mutator { return &processInclude{ fullPath: fullPath, relPath: relPath, @@ -23,10 +25,10 @@ func (m *processInclude) Name() string { return fmt.Sprintf("ProcessInclude(%s)", m.relPath) } -func (m *processInclude) Apply(root *config.Root) ([]Mutator, error) { +func (m *processInclude) Apply(_ context.Context, b *bundle.Bundle) ([]bundle.Mutator, error) { this, err := config.Load(m.fullPath) if err != nil { return nil, err } - return nil, root.Merge(this) + return nil, b.Config.Merge(this) } diff --git a/bundle/config/mutator/process_include_test.go b/bundle/config/mutator/process_include_test.go index 70138692..cb5348a2 100644 --- a/bundle/config/mutator/process_include_test.go +++ b/bundle/config/mutator/process_include_test.go @@ -1,11 +1,13 @@ package mutator_test import ( + "context" "fmt" "os" "path/filepath" "testing" + "github.com/databricks/bricks/bundle" "github.com/databricks/bricks/bundle/config" "github.com/databricks/bricks/bundle/config/mutator" "github.com/stretchr/testify/assert" @@ -13,22 +15,24 @@ import ( ) func TestProcessInclude(t *testing.T) { - root := &config.Root{ - Path: t.TempDir(), - Workspace: config.Workspace{ - Host: "foo", + bundle := &bundle.Bundle{ + Config: config.Root{ + Path: t.TempDir(), + Workspace: config.Workspace{ + Host: "foo", + }, }, } relPath := "./file.yml" - fullPath := filepath.Join(root.Path, relPath) + fullPath := filepath.Join(bundle.Config.Path, relPath) f, err := os.Create(fullPath) require.NoError(t, err) fmt.Fprint(f, "workspace:\n host: bar\n") f.Close() - assert.Equal(t, "foo", root.Workspace.Host) - _, err = mutator.ProcessInclude(fullPath, relPath).Apply(root) + assert.Equal(t, "foo", bundle.Config.Workspace.Host) + _, err = mutator.ProcessInclude(fullPath, relPath).Apply(context.Background(), bundle) require.NoError(t, err) - assert.Equal(t, "bar", root.Workspace.Host) + assert.Equal(t, "bar", bundle.Config.Workspace.Host) } diff --git a/bundle/config/mutator/process_root_includes.go b/bundle/config/mutator/process_root_includes.go index 32f32cb1..b3986406 100644 --- a/bundle/config/mutator/process_root_includes.go +++ b/bundle/config/mutator/process_root_includes.go @@ -1,9 +1,11 @@ package mutator import ( + "context" "fmt" "path/filepath" + "github.com/databricks/bricks/bundle" "github.com/databricks/bricks/bundle/config" "golang.org/x/exp/slices" ) @@ -12,7 +14,7 @@ type processRootIncludes struct{} // ProcessRootIncludes expands the patterns in the configuration's include list // into a list of mutators for each matching file. -func ProcessRootIncludes() Mutator { +func ProcessRootIncludes() bundle.Mutator { return &processRootIncludes{} } @@ -20,8 +22,8 @@ func (m *processRootIncludes) Name() string { return "ProcessRootIncludes" } -func (m *processRootIncludes) Apply(root *config.Root) ([]Mutator, error) { - var out []Mutator +func (m *processRootIncludes) Apply(_ context.Context, b *bundle.Bundle) ([]bundle.Mutator, error) { + var out []bundle.Mutator // Map with files we've already seen to avoid loading them twice. var seen = map[string]bool{ @@ -31,14 +33,14 @@ func (m *processRootIncludes) Apply(root *config.Root) ([]Mutator, error) { // For each glob, find all files to load. // Ordering of the list of globs is maintained in the output. // For matches that appear in multiple globs, only the first is kept. - for _, entry := range root.Include { + for _, entry := range b.Config.Include { // Include paths must be relative. if filepath.IsAbs(entry) { return nil, fmt.Errorf("%s: includes must be relative paths", entry) } // Anchor includes to the bundle root path. - matches, err := filepath.Glob(filepath.Join(root.Path, entry)) + matches, err := filepath.Glob(filepath.Join(b.Config.Path, entry)) if err != nil { return nil, err } @@ -46,7 +48,7 @@ func (m *processRootIncludes) Apply(root *config.Root) ([]Mutator, error) { // Filter matches to ones we haven't seen yet. var includes []string for _, match := range matches { - rel, err := filepath.Rel(root.Path, match) + rel, err := filepath.Rel(b.Config.Path, match) if err != nil { return nil, err } @@ -60,7 +62,7 @@ func (m *processRootIncludes) Apply(root *config.Root) ([]Mutator, error) { // Add matches to list of mutators to return. slices.Sort(includes) for _, include := range includes { - out = append(out, ProcessInclude(filepath.Join(root.Path, include), include)) + out = append(out, ProcessInclude(filepath.Join(b.Config.Path, include), include)) } } diff --git a/bundle/config/mutator/process_root_includes_test.go b/bundle/config/mutator/process_root_includes_test.go index de112b7d..ef1571cc 100644 --- a/bundle/config/mutator/process_root_includes_test.go +++ b/bundle/config/mutator/process_root_includes_test.go @@ -1,10 +1,12 @@ package mutator_test import ( + "context" "os" "path/filepath" "testing" + "github.com/databricks/bricks/bundle" "github.com/databricks/bricks/bundle/config" "github.com/databricks/bricks/bundle/config/mutator" "github.com/stretchr/testify/assert" @@ -18,36 +20,44 @@ func touch(t *testing.T, path, file string) { } func TestProcessRootIncludesEmpty(t *testing.T) { - root := &config.Root{Path: "."} - _, err := mutator.ProcessRootIncludes().Apply(root) + bundle := &bundle.Bundle{ + Config: config.Root{ + Path: ".", + }, + } + _, err := mutator.ProcessRootIncludes().Apply(context.Background(), bundle) require.NoError(t, err) } func TestProcessRootIncludesAbs(t *testing.T) { - root := &config.Root{ - Path: ".", - Include: []string{ - "/tmp/*.yml", + bundle := &bundle.Bundle{ + Config: config.Root{ + Path: ".", + Include: []string{ + "/tmp/*.yml", + }, }, } - _, err := mutator.ProcessRootIncludes().Apply(root) + _, err := mutator.ProcessRootIncludes().Apply(context.Background(), bundle) require.Error(t, err) assert.Contains(t, err.Error(), "must be relative paths") } func TestProcessRootIncludesSingleGlob(t *testing.T) { - root := &config.Root{ - Path: t.TempDir(), - Include: []string{ - "*.yml", + bundle := &bundle.Bundle{ + Config: config.Root{ + Path: t.TempDir(), + Include: []string{ + "*.yml", + }, }, } - touch(t, root.Path, "bundle.yml") - touch(t, root.Path, "a.yml") - touch(t, root.Path, "b.yml") + touch(t, bundle.Config.Path, "bundle.yml") + touch(t, bundle.Config.Path, "a.yml") + touch(t, bundle.Config.Path, "b.yml") - ms, err := mutator.ProcessRootIncludes().Apply(root) + ms, err := mutator.ProcessRootIncludes().Apply(context.Background(), bundle) require.NoError(t, err) var names []string @@ -61,18 +71,20 @@ func TestProcessRootIncludesSingleGlob(t *testing.T) { } func TestProcessRootIncludesMultiGlob(t *testing.T) { - root := &config.Root{ - Path: t.TempDir(), - Include: []string{ - "a*.yml", - "b*.yml", + bundle := &bundle.Bundle{ + Config: config.Root{ + Path: t.TempDir(), + Include: []string{ + "a*.yml", + "b*.yml", + }, }, } - touch(t, root.Path, "a1.yml") - touch(t, root.Path, "b1.yml") + touch(t, bundle.Config.Path, "a1.yml") + touch(t, bundle.Config.Path, "b1.yml") - ms, err := mutator.ProcessRootIncludes().Apply(root) + ms, err := mutator.ProcessRootIncludes().Apply(context.Background(), bundle) require.NoError(t, err) var names []string @@ -85,17 +97,19 @@ func TestProcessRootIncludesMultiGlob(t *testing.T) { } func TestProcessRootIncludesRemoveDups(t *testing.T) { - root := &config.Root{ - Path: t.TempDir(), - Include: []string{ - "*.yml", - "*.yml", + bundle := &bundle.Bundle{ + Config: config.Root{ + Path: t.TempDir(), + Include: []string{ + "*.yml", + "*.yml", + }, }, } - touch(t, root.Path, "a.yml") + touch(t, bundle.Config.Path, "a.yml") - ms, err := mutator.ProcessRootIncludes().Apply(root) + ms, err := mutator.ProcessRootIncludes().Apply(context.Background(), bundle) require.NoError(t, err) assert.Len(t, ms, 1) assert.Equal(t, "ProcessInclude(a.yml)", ms[0].Name()) diff --git a/bundle/config/mutator/select_environment.go b/bundle/config/mutator/select_environment.go index 1ceea4de..49d68547 100644 --- a/bundle/config/mutator/select_environment.go +++ b/bundle/config/mutator/select_environment.go @@ -1,9 +1,10 @@ package mutator import ( + "context" "fmt" - "github.com/databricks/bricks/bundle/config" + "github.com/databricks/bricks/bundle" ) type selectEnvironment struct { @@ -11,7 +12,7 @@ type selectEnvironment struct { } // SelectEnvironment merges the specified environment into the root configuration. -func SelectEnvironment(name string) Mutator { +func SelectEnvironment(name string) bundle.Mutator { return &selectEnvironment{ name: name, } @@ -21,27 +22,27 @@ func (m *selectEnvironment) Name() string { return fmt.Sprintf("SelectEnvironment(%s)", m.name) } -func (m *selectEnvironment) Apply(root *config.Root) ([]Mutator, error) { - if root.Environments == nil { +func (m *selectEnvironment) Apply(_ context.Context, b *bundle.Bundle) ([]bundle.Mutator, error) { + if b.Config.Environments == nil { return nil, fmt.Errorf("no environments defined") } // Get specified environment - env, ok := root.Environments[m.name] + env, ok := b.Config.Environments[m.name] if !ok { return nil, fmt.Errorf("%s: no such environment", m.name) } // Merge specified environment into root configuration structure. - err := root.MergeEnvironment(env) + err := b.Config.MergeEnvironment(env) if err != nil { return nil, err } // Store specified environment in configuration for reference. - root.Bundle.Environment = m.name + b.Config.Bundle.Environment = m.name // Clear environments after loading. - root.Environments = nil + b.Config.Environments = nil return nil, nil } diff --git a/bundle/config/mutator/select_environment_test.go b/bundle/config/mutator/select_environment_test.go index eaa98b18..ed91647f 100644 --- a/bundle/config/mutator/select_environment_test.go +++ b/bundle/config/mutator/select_environment_test.go @@ -1,8 +1,10 @@ package mutator_test import ( + "context" "testing" + "github.com/databricks/bricks/bundle" "github.com/databricks/bricks/bundle/config" "github.com/databricks/bricks/bundle/config/mutator" "github.com/stretchr/testify/assert" @@ -10,29 +12,33 @@ import ( ) func TestSelectEnvironment(t *testing.T) { - root := &config.Root{ - Workspace: config.Workspace{ - Host: "foo", - }, - Environments: map[string]*config.Environment{ - "default": { - Workspace: &config.Workspace{ - Host: "bar", + bundle := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + Host: "foo", + }, + Environments: map[string]*config.Environment{ + "default": { + Workspace: &config.Workspace{ + Host: "bar", + }, }, }, }, } - _, err := mutator.SelectEnvironment("default").Apply(root) + _, err := mutator.SelectEnvironment("default").Apply(context.Background(), bundle) require.NoError(t, err) - assert.Equal(t, "bar", root.Workspace.Host) + assert.Equal(t, "bar", bundle.Config.Workspace.Host) } func TestSelectEnvironmentNotFound(t *testing.T) { - root := &config.Root{ - Environments: map[string]*config.Environment{ - "default": {}, + bundle := &bundle.Bundle{ + Config: config.Root{ + Environments: map[string]*config.Environment{ + "default": {}, + }, }, } - _, err := mutator.SelectEnvironment("doesnt-exist").Apply(root) + _, err := mutator.SelectEnvironment("doesnt-exist").Apply(context.Background(), bundle) require.Error(t, err, "no environments defined") } diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index 596b57da..fbec1e65 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -25,7 +25,7 @@ func TestRootMarshalUnmarshal(t *testing.T) { func TestRootLoad(t *testing.T) { root := &Root{} - err := root.Load("./tests/basic/bundle.yml") + err := root.Load("../tests/basic/bundle.yml") require.NoError(t, err) assert.Equal(t, "basic", root.Bundle.Name) } diff --git a/bundle/config/tests/environment_overrides_test.go b/bundle/config/tests/environment_overrides_test.go deleted file mode 100644 index 9fbf235f..00000000 --- a/bundle/config/tests/environment_overrides_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package config_tests - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestEnvironmentOverridesDev(t *testing.T) { - development := loadEnvironment(t, "./environment_overrides", "development") - assert.Equal(t, "https://development.acme.cloud.databricks.com/", development.Workspace.Host) - staging := loadEnvironment(t, "./environment_overrides", "staging") - assert.Equal(t, "https://staging.acme.cloud.databricks.com/", staging.Workspace.Host) -} diff --git a/bundle/config/tests/loader.go b/bundle/config/tests/loader.go deleted file mode 100644 index 1cc5ffc3..00000000 --- a/bundle/config/tests/loader.go +++ /dev/null @@ -1,24 +0,0 @@ -package config_tests - -import ( - "testing" - - "github.com/databricks/bricks/bundle/config" - "github.com/databricks/bricks/bundle/config/mutator" - "github.com/stretchr/testify/require" -) - -func load(t *testing.T, path string) *config.Root { - root, err := config.Load(path) - require.NoError(t, err) - err = mutator.Apply(root, mutator.DefaultMutators()) - require.NoError(t, err) - return root -} - -func loadEnvironment(t *testing.T, path, env string) *config.Root { - root := load(t, path) - err := mutator.Apply(root, []mutator.Mutator{mutator.SelectEnvironment(env)}) - require.NoError(t, err) - return root -} diff --git a/bundle/loader/loader.go b/bundle/loader/loader.go new file mode 100644 index 00000000..29c11d0f --- /dev/null +++ b/bundle/loader/loader.go @@ -0,0 +1,22 @@ +package loader + +import ( + "context" + + "github.com/databricks/bricks/bundle" + "github.com/databricks/bricks/bundle/config/mutator" +) + +func ConfigureForEnvironment(ctx context.Context, env string) (context.Context, error) { + b, err := bundle.LoadFromRoot() + if err != nil { + return nil, err + } + + err = bundle.Apply(ctx, b, mutator.DefaultMutatorsForEnvironment(env)) + if err != nil { + return nil, err + } + + return bundle.Context(ctx, b), nil +} diff --git a/bundle/mutator.go b/bundle/mutator.go new file mode 100644 index 00000000..8b3a970e --- /dev/null +++ b/bundle/mutator.go @@ -0,0 +1,36 @@ +package bundle + +import ( + "context" +) + +// Mutator is the interface type that mutates a bundle's configuration or internal state. +// This makes every mutation or action observable and debuggable. +type Mutator interface { + // Name returns the mutators name. + Name() string + + // Apply mutates the specified bundle object. + // It may return a list of mutators to apply immediately after this mutator. + // For example: when processing all configuration files in the tree; each file gets + // its own mutator instance. + Apply(context.Context, *Bundle) ([]Mutator, error) +} + +func Apply(ctx context.Context, b *Bundle, ms []Mutator) error { + if len(ms) == 0 { + return nil + } + for _, m := range ms { + ms_, err := m.Apply(ctx, b) + if err != nil { + return err + } + // Apply recursively. + err = Apply(ctx, b, ms_) + if err != nil { + return err + } + } + return nil +} diff --git a/bundle/mutator_test.go b/bundle/mutator_test.go new file mode 100644 index 00000000..8fb04254 --- /dev/null +++ b/bundle/mutator_test.go @@ -0,0 +1,44 @@ +package bundle + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +type testMutator struct { + applyCalled int + nestedMutators []Mutator +} + +func (t *testMutator) Name() string { + return "test" +} + +func (t *testMutator) Apply(_ context.Context, b *Bundle) ([]Mutator, error) { + t.applyCalled++ + return t.nestedMutators, nil +} + +func TestMutator(t *testing.T) { + nested := []*testMutator{ + {}, + {}, + } + + m := &testMutator{ + nestedMutators: []Mutator{ + nested[0], + nested[1], + }, + } + + bundle := &Bundle{} + err := Apply(context.Background(), bundle, []Mutator{m}) + assert.NoError(t, err) + + assert.Equal(t, 1, m.applyCalled) + assert.Equal(t, 1, nested[0].applyCalled) + assert.Equal(t, 1, nested[1].applyCalled) +} diff --git a/bundle/config/tests/README.md b/bundle/tests/README.md similarity index 100% rename from bundle/config/tests/README.md rename to bundle/tests/README.md diff --git a/bundle/config/tests/basic/bundle.yml b/bundle/tests/basic/bundle.yml similarity index 100% rename from bundle/config/tests/basic/bundle.yml rename to bundle/tests/basic/bundle.yml diff --git a/bundle/config/tests/basic_test.go b/bundle/tests/basic_test.go similarity index 61% rename from bundle/config/tests/basic_test.go rename to bundle/tests/basic_test.go index 2a078594..cd6100fd 100644 --- a/bundle/config/tests/basic_test.go +++ b/bundle/tests/basic_test.go @@ -7,6 +7,6 @@ import ( ) func TestBasic(t *testing.T) { - root := load(t, "./basic") - assert.Equal(t, "basic", root.Bundle.Name) + b := load(t, "./basic") + assert.Equal(t, "basic", b.Config.Bundle.Name) } diff --git a/bundle/config/tests/environment_overrides/bundle.yml b/bundle/tests/environment_overrides/bundle.yml similarity index 100% rename from bundle/config/tests/environment_overrides/bundle.yml rename to bundle/tests/environment_overrides/bundle.yml diff --git a/bundle/tests/environment_overrides_test.go b/bundle/tests/environment_overrides_test.go new file mode 100644 index 00000000..4b8401c8 --- /dev/null +++ b/bundle/tests/environment_overrides_test.go @@ -0,0 +1,17 @@ +package config_tests + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEnvironmentOverridesDev(t *testing.T) { + b := loadEnvironment(t, "./environment_overrides", "development") + assert.Equal(t, "https://development.acme.cloud.databricks.com/", b.Config.Workspace.Host) +} + +func TestEnvironmentOverridesStaging(t *testing.T) { + b := loadEnvironment(t, "./environment_overrides", "staging") + assert.Equal(t, "https://staging.acme.cloud.databricks.com/", b.Config.Workspace.Host) +} diff --git a/bundle/config/tests/include_default/bundle.yml b/bundle/tests/include_default/bundle.yml similarity index 100% rename from bundle/config/tests/include_default/bundle.yml rename to bundle/tests/include_default/bundle.yml diff --git a/bundle/config/tests/include_default/my_first_job/resource.yml b/bundle/tests/include_default/my_first_job/resource.yml similarity index 100% rename from bundle/config/tests/include_default/my_first_job/resource.yml rename to bundle/tests/include_default/my_first_job/resource.yml diff --git a/bundle/config/tests/include_default/my_second_job/resource.yml b/bundle/tests/include_default/my_second_job/resource.yml similarity index 100% rename from bundle/config/tests/include_default/my_second_job/resource.yml rename to bundle/tests/include_default/my_second_job/resource.yml diff --git a/bundle/config/tests/include_default_test.go b/bundle/tests/include_default_test.go similarity index 57% rename from bundle/config/tests/include_default_test.go rename to bundle/tests/include_default_test.go index e154b69a..3ef0d8bb 100644 --- a/bundle/config/tests/include_default_test.go +++ b/bundle/tests/include_default_test.go @@ -9,12 +9,12 @@ import ( ) func TestIncludeDefault(t *testing.T) { - root := load(t, "./include_default") + b := load(t, "./include_default") // Test that both jobs were loaded. - keys := maps.Keys(root.Resources.Jobs) + keys := maps.Keys(b.Config.Resources.Jobs) sort.Strings(keys) assert.Equal(t, []string{"my_first_job", "my_second_job"}, keys) - assert.Equal(t, "1", root.Resources.Jobs["my_first_job"].ID) - assert.Equal(t, "2", root.Resources.Jobs["my_second_job"].ID) + assert.Equal(t, "1", b.Config.Resources.Jobs["my_first_job"].ID) + assert.Equal(t, "2", b.Config.Resources.Jobs["my_second_job"].ID) } diff --git a/bundle/config/tests/include_override/bundle.yml b/bundle/tests/include_override/bundle.yml similarity index 100% rename from bundle/config/tests/include_override/bundle.yml rename to bundle/tests/include_override/bundle.yml diff --git a/bundle/config/tests/include_override/this_file_isnt_included.yml b/bundle/tests/include_override/this_file_isnt_included.yml similarity index 100% rename from bundle/config/tests/include_override/this_file_isnt_included.yml rename to bundle/tests/include_override/this_file_isnt_included.yml diff --git a/bundle/config/tests/include_override_test.go b/bundle/tests/include_override_test.go similarity index 61% rename from bundle/config/tests/include_override_test.go rename to bundle/tests/include_override_test.go index 8779c6b6..0e18fab3 100644 --- a/bundle/config/tests/include_override_test.go +++ b/bundle/tests/include_override_test.go @@ -7,6 +7,6 @@ import ( ) func TestIncludeOverride(t *testing.T) { - root := load(t, "./include_override") - assert.Empty(t, root.Resources.Jobs) + b := load(t, "./include_override") + assert.Empty(t, b.Config.Resources.Jobs) } diff --git a/bundle/config/tests/job_and_pipeline/bundle.yml b/bundle/tests/job_and_pipeline/bundle.yml similarity index 100% rename from bundle/config/tests/job_and_pipeline/bundle.yml rename to bundle/tests/job_and_pipeline/bundle.yml diff --git a/bundle/config/tests/job_and_pipeline_test.go b/bundle/tests/job_and_pipeline_test.go similarity index 59% rename from bundle/config/tests/job_and_pipeline_test.go rename to bundle/tests/job_and_pipeline_test.go index 809cb3c0..677bed82 100644 --- a/bundle/config/tests/job_and_pipeline_test.go +++ b/bundle/tests/job_and_pipeline_test.go @@ -8,11 +8,11 @@ import ( ) func TestJobAndPipelineDevelopment(t *testing.T) { - root := loadEnvironment(t, "./job_and_pipeline", "development") - assert.Len(t, root.Resources.Jobs, 0) - assert.Len(t, root.Resources.Pipelines, 1) + b := loadEnvironment(t, "./job_and_pipeline", "development") + assert.Len(t, b.Config.Resources.Jobs, 0) + assert.Len(t, b.Config.Resources.Pipelines, 1) - p := root.Resources.Pipelines["nyc_taxi_pipeline"] + p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] assert.True(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) @@ -20,11 +20,11 @@ func TestJobAndPipelineDevelopment(t *testing.T) { } func TestJobAndPipelineStaging(t *testing.T) { - root := loadEnvironment(t, "./job_and_pipeline", "staging") - assert.Len(t, root.Resources.Jobs, 0) - assert.Len(t, root.Resources.Pipelines, 1) + b := loadEnvironment(t, "./job_and_pipeline", "staging") + assert.Len(t, b.Config.Resources.Jobs, 0) + assert.Len(t, b.Config.Resources.Pipelines, 1) - p := root.Resources.Pipelines["nyc_taxi_pipeline"] + p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) @@ -32,17 +32,17 @@ func TestJobAndPipelineStaging(t *testing.T) { } func TestJobAndPipelineProduction(t *testing.T) { - root := loadEnvironment(t, "./job_and_pipeline", "production") - assert.Len(t, root.Resources.Jobs, 1) - assert.Len(t, root.Resources.Pipelines, 1) + b := loadEnvironment(t, "./job_and_pipeline", "production") + assert.Len(t, b.Config.Resources.Jobs, 1) + assert.Len(t, b.Config.Resources.Pipelines, 1) - p := root.Resources.Pipelines["nyc_taxi_pipeline"] + p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) assert.Equal(t, "nyc_taxi_production", p.Target) - j := root.Resources.Jobs["pipeline_schedule"] + j := b.Config.Resources.Jobs["pipeline_schedule"] assert.Equal(t, "Daily refresh of production pipeline", j.Name) require.Len(t, j.Tasks, 1) assert.NotEmpty(t, j.Tasks[0].PipelineTask.PipelineId) diff --git a/bundle/tests/loader.go b/bundle/tests/loader.go new file mode 100644 index 00000000..fcf3ffb1 --- /dev/null +++ b/bundle/tests/loader.go @@ -0,0 +1,25 @@ +package config_tests + +import ( + "context" + "testing" + + "github.com/databricks/bricks/bundle" + "github.com/databricks/bricks/bundle/config/mutator" + "github.com/stretchr/testify/require" +) + +func load(t *testing.T, path string) *bundle.Bundle { + b, err := bundle.Load(path) + require.NoError(t, err) + err = bundle.Apply(context.Background(), b, mutator.DefaultMutators()) + require.NoError(t, err) + return b +} + +func loadEnvironment(t *testing.T, path, env string) *bundle.Bundle { + b := load(t, path) + err := bundle.Apply(context.Background(), b, []bundle.Mutator{mutator.SelectEnvironment(env)}) + require.NoError(t, err) + return b +} diff --git a/bundle/config/tests/yaml_anchors/bundle.yml b/bundle/tests/yaml_anchors/bundle.yml similarity index 100% rename from bundle/config/tests/yaml_anchors/bundle.yml rename to bundle/tests/yaml_anchors/bundle.yml diff --git a/bundle/config/tests/yaml_anchors_test.go b/bundle/tests/yaml_anchors_test.go similarity index 77% rename from bundle/config/tests/yaml_anchors_test.go rename to bundle/tests/yaml_anchors_test.go index fb560d10..95cec30a 100644 --- a/bundle/config/tests/yaml_anchors_test.go +++ b/bundle/tests/yaml_anchors_test.go @@ -8,10 +8,10 @@ import ( ) func TestYAMLAnchors(t *testing.T) { - root := load(t, "./yaml_anchors") - assert.Len(t, root.Resources.Jobs, 1) + b := load(t, "./yaml_anchors") + assert.Len(t, b.Config.Resources.Jobs, 1) - j := root.Resources.Jobs["my_job"] + j := b.Config.Resources.Jobs["my_job"] require.Len(t, j.Tasks, 2) t0 := j.Tasks[0] diff --git a/cmd/bundle/root.go b/cmd/bundle/root.go index 7326f1bb..b01f56c6 100644 --- a/cmd/bundle/root.go +++ b/cmd/bundle/root.go @@ -1,7 +1,7 @@ package bundle import ( - "github.com/databricks/bricks/bundle" + "github.com/databricks/bricks/bundle/loader" "github.com/databricks/bricks/cmd/root" "github.com/spf13/cobra" ) @@ -15,7 +15,7 @@ var rootCmd = &cobra.Command{ // ConfigureBundle loads the bundle configuration // and configures it on the command's context. func ConfigureBundle(cmd *cobra.Command, args []string) error { - ctx, err := bundle.ConfigureForEnvironment(cmd.Context(), getEnvironment(cmd)) + ctx, err := loader.ConfigureForEnvironment(cmd.Context(), getEnvironment(cmd)) if err != nil { return err }