Renamed `environments` to `targets` in bundle configuration (#670)

## Changes
Renamed Environments to Targets in bundle.yml.

The change is backward-compatible and customers can continue to use
`environments` in the time being.

## Tests
Added tests which checks that both `environments` and `targets` sections
in bundle.yml works correctly
This commit is contained in:
Andrew Nester 2023-08-17 17:22:32 +02:00 committed by GitHub
parent 4694832534
commit 56dcd3f0a7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
63 changed files with 768 additions and 416 deletions

View File

@ -117,10 +117,10 @@ func (b *Bundle) WorkspaceClient() *databricks.WorkspaceClient {
}
// CacheDir returns directory to use for temporary files for this bundle.
// Scoped to the bundle's environment.
// Scoped to the bundle's target.
func (b *Bundle) CacheDir(paths ...string) (string, error) {
if b.Config.Bundle.Environment == "" {
panic("environment not set")
if b.Config.Bundle.Target == "" {
panic("target not set")
}
cacheDirName, exists := os.LookupEnv("DATABRICKS_BUNDLE_TMP")
@ -138,8 +138,8 @@ func (b *Bundle) CacheDir(paths ...string) (string, error) {
// Fixed components of the result path.
parts := []string{
cacheDirName,
// Scope with environment name.
b.Config.Bundle.Environment,
// Scope with target name.
b.Config.Bundle.Target,
}
// Append dynamic components of the result path.

View File

@ -31,16 +31,16 @@ func TestBundleCacheDir(t *testing.T) {
bundle, err := Load(context.Background(), projectDir)
require.NoError(t, err)
// Artificially set environment.
// This is otherwise done by [mutators.SelectEnvironment].
bundle.Config.Bundle.Environment = "default"
// Artificially set target.
// This is otherwise done by [mutators.SelectTarget].
bundle.Config.Bundle.Target = "default"
// unset env variable in case it's set
t.Setenv("DATABRICKS_BUNDLE_TMP", "")
cacheDir, err := bundle.CacheDir()
// format is <CWD>/.databricks/bundle/<environment>
// format is <CWD>/.databricks/bundle/<target>
assert.NoError(t, err)
assert.Equal(t, filepath.Join(projectDir, ".databricks", "bundle", "default"), cacheDir)
}
@ -55,16 +55,16 @@ func TestBundleCacheDirOverride(t *testing.T) {
bundle, err := Load(context.Background(), projectDir)
require.NoError(t, err)
// Artificially set environment.
// This is otherwise done by [mutators.SelectEnvironment].
bundle.Config.Bundle.Environment = "default"
// Artificially set target.
// This is otherwise done by [mutators.SelectTarget].
bundle.Config.Bundle.Target = "default"
// now we expect to use 'bundleTmpDir' instead of CWD/.databricks/bundle
t.Setenv("DATABRICKS_BUNDLE_TMP", bundleTmpDir)
cacheDir, err := bundle.CacheDir()
// format is <DATABRICKS_BUNDLE_TMP>/<environment>
// format is <DATABRICKS_BUNDLE_TMP>/<target>
assert.NoError(t, err)
assert.Equal(t, filepath.Join(bundleTmpDir, "default"), cacheDir)
}

View File

@ -15,7 +15,10 @@ type Bundle struct {
// Default warehouse to run SQL on.
// DefaultWarehouse string `json:"default_warehouse,omitempty"`
// Environment is set by the mutator that selects the environment.
// Target is set by the mutator that selects the target.
Target string `json:"target,omitempty" bundle:"readonly"`
// DEPRECATED. Left for backward compatibility with Target
Environment string `json:"environment,omitempty" bundle:"readonly"`
// Terraform holds configuration related to Terraform.
@ -32,10 +35,10 @@ type Bundle struct {
// origin url. Automatically loaded by reading .git directory if not specified
Git Git `json:"git,omitempty"`
// Determines the mode of the environment.
// Determines the mode of the target.
// For example, 'mode: development' can be used for deployments for
// development purposes.
// Annotated readonly as this should be set at the environment level.
// Annotated readonly as this should be set at the target level.
Mode Mode `json:"mode,omitempty" bundle:"readonly"`
// Overrides the compute used for jobs and other supported assets.

View File

@ -1,37 +0,0 @@
package mutator
import (
"context"
"fmt"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
)
type defineDefaultEnvironment struct {
name string
}
// DefineDefaultEnvironment adds an environment named "default"
// to the configuration if none have been defined.
func DefineDefaultEnvironment() bundle.Mutator {
return &defineDefaultEnvironment{
name: "default",
}
}
func (m *defineDefaultEnvironment) Name() string {
return fmt.Sprintf("DefineDefaultEnvironment(%s)", m.name)
}
func (m *defineDefaultEnvironment) Apply(_ context.Context, b *bundle.Bundle) error {
// Nothing to do if the configuration has at least 1 environment.
if len(b.Config.Environments) > 0 {
return nil
}
// Define default environment.
b.Config.Environments = make(map[string]*config.Environment)
b.Config.Environments[m.name] = &config.Environment{}
return nil
}

View File

@ -0,0 +1,37 @@
package mutator
import (
"context"
"fmt"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
)
type defineDefaultTarget struct {
name string
}
// DefineDefaultTarget adds a target named "default"
// to the configuration if none have been defined.
func DefineDefaultTarget() bundle.Mutator {
return &defineDefaultTarget{
name: "default",
}
}
func (m *defineDefaultTarget) Name() string {
return fmt.Sprintf("DefineDefaultTarget(%s)", m.name)
}
func (m *defineDefaultTarget) Apply(_ context.Context, b *bundle.Bundle) error {
// Nothing to do if the configuration has at least 1 target.
if len(b.Config.Targets) > 0 {
return nil
}
// Define default target.
b.Config.Targets = make(map[string]*config.Target)
b.Config.Targets[m.name] = &config.Target{}
return nil
}

View File

@ -11,25 +11,25 @@ import (
"github.com/stretchr/testify/require"
)
func TestDefaultEnvironment(t *testing.T) {
func TestDefaultTarget(t *testing.T) {
bundle := &bundle.Bundle{}
err := mutator.DefineDefaultEnvironment().Apply(context.Background(), bundle)
err := mutator.DefineDefaultTarget().Apply(context.Background(), bundle)
require.NoError(t, err)
env, ok := bundle.Config.Environments["default"]
env, ok := bundle.Config.Targets["default"]
assert.True(t, ok)
assert.Equal(t, &config.Environment{}, env)
assert.Equal(t, &config.Target{}, env)
}
func TestDefaultEnvironmentAlreadySpecified(t *testing.T) {
func TestDefaultTargetAlreadySpecified(t *testing.T) {
bundle := &bundle.Bundle{
Config: config.Root{
Environments: map[string]*config.Environment{
Targets: map[string]*config.Target{
"development": {},
},
},
}
err := mutator.DefineDefaultEnvironment().Apply(context.Background(), bundle)
err := mutator.DefineDefaultTarget().Apply(context.Background(), bundle)
require.NoError(t, err)
_, ok := bundle.Config.Environments["default"]
_, ok := bundle.Config.Targets["default"]
assert.False(t, ok)
}

View File

@ -27,14 +27,14 @@ func (m *defineDefaultWorkspaceRoot) Apply(ctx context.Context, b *bundle.Bundle
return fmt.Errorf("unable to define default workspace root: bundle name not defined")
}
if b.Config.Bundle.Environment == "" {
return fmt.Errorf("unable to define default workspace root: bundle environment not selected")
if b.Config.Bundle.Target == "" {
return fmt.Errorf("unable to define default workspace root: bundle target not selected")
}
b.Config.Workspace.RootPath = fmt.Sprintf(
"~/.bundle/%s/%s",
b.Config.Bundle.Name,
b.Config.Bundle.Environment,
b.Config.Bundle.Target,
)
return nil
}

View File

@ -16,7 +16,7 @@ func TestDefaultWorkspaceRoot(t *testing.T) {
Config: config.Root{
Bundle: config.Bundle{
Name: "name",
Environment: "environment",
Target: "environment",
},
},
}

View File

@ -7,11 +7,11 @@ import (
func DefaultMutators() []bundle.Mutator {
return []bundle.Mutator{
ProcessRootIncludes(),
DefineDefaultEnvironment(),
DefineDefaultTarget(),
LoadGitDetails(),
}
}
func DefaultMutatorsForEnvironment(env string) []bundle.Mutator {
return append(DefaultMutators(), SelectEnvironment(env))
func DefaultMutatorsForTarget(env string) []bundle.Mutator {
return append(DefaultMutators(), SelectTarget(env))
}

View File

@ -35,7 +35,7 @@ func overrideJobCompute(j *resources.Job, compute string) {
func (m *overrideCompute) Apply(ctx context.Context, b *bundle.Bundle) error {
if b.Config.Bundle.Mode != config.Development {
if b.Config.Bundle.ComputeID != "" {
return fmt.Errorf("cannot override compute for an environment that does not use 'mode: development'")
return fmt.Errorf("cannot override compute for an target that does not use 'mode: development'")
}
return nil
}

View File

@ -13,16 +13,16 @@ import (
"github.com/databricks/databricks-sdk-go/service/ml"
)
type processEnvironmentMode struct{}
type processTargetMode struct{}
const developmentConcurrentRuns = 4
func ProcessEnvironmentMode() bundle.Mutator {
return &processEnvironmentMode{}
func ProcessTargetMode() bundle.Mutator {
return &processTargetMode{}
}
func (m *processEnvironmentMode) Name() string {
return "ProcessEnvironmentMode"
func (m *processTargetMode) Name() string {
return "ProcessTargetMode"
}
// Mark all resources as being for 'development' purposes, i.e.
@ -110,14 +110,14 @@ func findIncorrectPath(b *bundle.Bundle, mode config.Mode) string {
func validateProductionMode(ctx context.Context, b *bundle.Bundle, isPrincipalUsed bool) error {
if b.Config.Bundle.Git.Inferred {
env := b.Config.Bundle.Environment
return fmt.Errorf("environment with 'mode: production' must specify an explicit 'environments.%s.git' configuration", env)
env := b.Config.Bundle.Target
return fmt.Errorf("target with 'mode: production' must specify an explicit 'targets.%s.git' configuration", env)
}
r := b.Config.Resources
for i := range r.Pipelines {
if r.Pipelines[i].Development {
return fmt.Errorf("environment with 'mode: production' cannot specify a pipeline with 'development: true'")
return fmt.Errorf("target with 'mode: production' cannot specify a pipeline with 'development: true'")
}
}
@ -125,7 +125,7 @@ func validateProductionMode(ctx context.Context, b *bundle.Bundle, isPrincipalUs
if path := findIncorrectPath(b, config.Production); path != "" {
message := "%s must not contain the current username when using 'mode: production'"
if path == "root_path" {
return fmt.Errorf(message+"\n tip: set workspace.root_path to a shared path such as /Shared/.bundle/${bundle.name}/${bundle.environment}", path)
return fmt.Errorf(message+"\n tip: set workspace.root_path to a shared path such as /Shared/.bundle/${bundle.name}/${bundle.target}", path)
} else {
return fmt.Errorf(message, path)
}
@ -165,7 +165,7 @@ func isRunAsSet(r config.Resources) bool {
return true
}
func (m *processEnvironmentMode) Apply(ctx context.Context, b *bundle.Bundle) error {
func (m *processTargetMode) Apply(ctx context.Context, b *bundle.Bundle) error {
switch b.Config.Bundle.Mode {
case config.Development:
err := validateDevelopmentMode(b)

View File

@ -58,10 +58,10 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
}
}
func TestProcessEnvironmentModeDevelopment(t *testing.T) {
func TestProcessTargetModeDevelopment(t *testing.T) {
bundle := mockBundle(config.Development)
m := ProcessEnvironmentMode()
m := ProcessTargetMode()
err := m.Apply(context.Background(), bundle)
require.NoError(t, err)
assert.Equal(t, "[dev lennart] job1", bundle.Config.Resources.Jobs["job1"].Name)
@ -73,10 +73,10 @@ func TestProcessEnvironmentModeDevelopment(t *testing.T) {
assert.True(t, bundle.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
}
func TestProcessEnvironmentModeDefault(t *testing.T) {
func TestProcessTargetModeDefault(t *testing.T) {
bundle := mockBundle("")
m := ProcessEnvironmentMode()
m := ProcessTargetMode()
err := m.Apply(context.Background(), bundle)
require.NoError(t, err)
assert.Equal(t, "job1", bundle.Config.Resources.Jobs["job1"].Name)
@ -84,7 +84,7 @@ func TestProcessEnvironmentModeDefault(t *testing.T) {
assert.False(t, bundle.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
}
func TestProcessEnvironmentModeProduction(t *testing.T) {
func TestProcessTargetModeProduction(t *testing.T) {
bundle := mockBundle(config.Production)
err := validateProductionMode(context.Background(), bundle, false)
@ -118,7 +118,7 @@ func TestProcessEnvironmentModeProduction(t *testing.T) {
assert.False(t, bundle.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
}
func TestProcessEnvironmentModeProductionGit(t *testing.T) {
func TestProcessTargetModeProductionGit(t *testing.T) {
bundle := mockBundle(config.Production)
// Pretend the user didn't set Git configuration explicitly
@ -129,10 +129,10 @@ func TestProcessEnvironmentModeProductionGit(t *testing.T) {
bundle.Config.Bundle.Git.Inferred = false
}
func TestProcessEnvironmentModeProductionOkForPrincipal(t *testing.T) {
func TestProcessTargetModeProductionOkForPrincipal(t *testing.T) {
bundle := mockBundle(config.Production)
// Our environment has all kinds of problems when not using service principals ...
// Our target has all kinds of problems when not using service principals ...
err := validateProductionMode(context.Background(), bundle, false)
require.Error(t, err)
@ -152,7 +152,7 @@ func TestAllResourcesMocked(t *testing.T) {
assert.True(
t,
!field.IsNil() && field.Len() > 0,
"process_environment_mode should support '%s' (please add it to process_environment_mode.go and extend the test suite)",
"process_target_mode should support '%s' (please add it to process_target_mode.go and extend the test suite)",
resources.Type().Field(i).Name,
)
}
@ -164,7 +164,7 @@ func TestAllResourcesRenamed(t *testing.T) {
bundle := mockBundle(config.Development)
resources := reflect.ValueOf(bundle.Config.Resources)
m := ProcessEnvironmentMode()
m := ProcessTargetMode()
err := m.Apply(context.Background(), bundle)
require.NoError(t, err)
@ -179,7 +179,7 @@ func TestAllResourcesRenamed(t *testing.T) {
assert.True(
t,
strings.Contains(nameField.String(), "dev"),
"process_environment_mode should rename '%s' in '%s'",
"process_target_mode should rename '%s' in '%s'",
key,
resources.Type().Field(i).Name,
)

View File

@ -1,54 +0,0 @@
package mutator
import (
"context"
"fmt"
"strings"
"github.com/databricks/cli/bundle"
"golang.org/x/exp/maps"
)
type selectDefaultEnvironment struct{}
// SelectDefaultEnvironment merges the default environment into the root configuration.
func SelectDefaultEnvironment() bundle.Mutator {
return &selectDefaultEnvironment{}
}
func (m *selectDefaultEnvironment) Name() string {
return "SelectDefaultEnvironment"
}
func (m *selectDefaultEnvironment) Apply(ctx context.Context, b *bundle.Bundle) error {
if len(b.Config.Environments) == 0 {
return fmt.Errorf("no environments defined")
}
// One environment means there's only one default.
names := maps.Keys(b.Config.Environments)
if len(names) == 1 {
return SelectEnvironment(names[0]).Apply(ctx, b)
}
// Multiple environments means we look for the `default` flag.
var defaults []string
for name, env := range b.Config.Environments {
if env != nil && env.Default {
defaults = append(defaults, name)
}
}
// It is invalid to have multiple environments with the `default` flag set.
if len(defaults) > 1 {
return fmt.Errorf("multiple environments are marked as default (%s)", strings.Join(defaults, ", "))
}
// If no environment has the `default` flag set, ask the user to specify one.
if len(defaults) == 0 {
return fmt.Errorf("please specify environment")
}
// One default remaining.
return SelectEnvironment(defaults[0]).Apply(ctx, b)
}

View File

@ -1,90 +0,0 @@
package mutator_test
import (
"context"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/stretchr/testify/assert"
)
func TestSelectDefaultEnvironmentNoEnvironments(t *testing.T) {
bundle := &bundle.Bundle{
Config: config.Root{
Environments: map[string]*config.Environment{},
},
}
err := mutator.SelectDefaultEnvironment().Apply(context.Background(), bundle)
assert.ErrorContains(t, err, "no environments defined")
}
func TestSelectDefaultEnvironmentSingleEnvironments(t *testing.T) {
bundle := &bundle.Bundle{
Config: config.Root{
Environments: map[string]*config.Environment{
"foo": {},
},
},
}
err := mutator.SelectDefaultEnvironment().Apply(context.Background(), bundle)
assert.NoError(t, err)
assert.Equal(t, "foo", bundle.Config.Bundle.Environment)
}
func TestSelectDefaultEnvironmentNoDefaults(t *testing.T) {
bundle := &bundle.Bundle{
Config: config.Root{
Environments: map[string]*config.Environment{
"foo": {},
"bar": {},
"qux": {},
},
},
}
err := mutator.SelectDefaultEnvironment().Apply(context.Background(), bundle)
assert.ErrorContains(t, err, "please specify environment")
}
func TestSelectDefaultEnvironmentNoDefaultsWithNil(t *testing.T) {
bundle := &bundle.Bundle{
Config: config.Root{
Environments: map[string]*config.Environment{
"foo": nil,
"bar": nil,
},
},
}
err := mutator.SelectDefaultEnvironment().Apply(context.Background(), bundle)
assert.ErrorContains(t, err, "please specify environment")
}
func TestSelectDefaultEnvironmentMultipleDefaults(t *testing.T) {
bundle := &bundle.Bundle{
Config: config.Root{
Environments: map[string]*config.Environment{
"foo": {Default: true},
"bar": {Default: true},
"qux": {Default: true},
},
},
}
err := mutator.SelectDefaultEnvironment().Apply(context.Background(), bundle)
assert.ErrorContains(t, err, "multiple environments are marked as default")
}
func TestSelectDefaultEnvironmentSingleDefault(t *testing.T) {
bundle := &bundle.Bundle{
Config: config.Root{
Environments: map[string]*config.Environment{
"foo": {},
"bar": {Default: true},
"qux": {},
},
},
}
err := mutator.SelectDefaultEnvironment().Apply(context.Background(), bundle)
assert.NoError(t, err)
assert.Equal(t, "bar", bundle.Config.Bundle.Environment)
}

View File

@ -0,0 +1,54 @@
package mutator
import (
"context"
"fmt"
"strings"
"github.com/databricks/cli/bundle"
"golang.org/x/exp/maps"
)
type selectDefaultTarget struct{}
// SelectDefaultTarget merges the default target into the root configuration.
func SelectDefaultTarget() bundle.Mutator {
return &selectDefaultTarget{}
}
func (m *selectDefaultTarget) Name() string {
return "SelectDefaultTarget"
}
func (m *selectDefaultTarget) Apply(ctx context.Context, b *bundle.Bundle) error {
if len(b.Config.Targets) == 0 {
return fmt.Errorf("no targets defined")
}
// One target means there's only one default.
names := maps.Keys(b.Config.Targets)
if len(names) == 1 {
return SelectTarget(names[0]).Apply(ctx, b)
}
// Multiple targets means we look for the `default` flag.
var defaults []string
for name, env := range b.Config.Targets {
if env != nil && env.Default {
defaults = append(defaults, name)
}
}
// It is invalid to have multiple targets with the `default` flag set.
if len(defaults) > 1 {
return fmt.Errorf("multiple targets are marked as default (%s)", strings.Join(defaults, ", "))
}
// If no target has the `default` flag set, ask the user to specify one.
if len(defaults) == 0 {
return fmt.Errorf("please specify target")
}
// One default remaining.
return SelectTarget(defaults[0]).Apply(ctx, b)
}

View File

@ -0,0 +1,90 @@
package mutator_test
import (
"context"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/stretchr/testify/assert"
)
func TestSelectDefaultTargetNoTargets(t *testing.T) {
bundle := &bundle.Bundle{
Config: config.Root{
Targets: map[string]*config.Target{},
},
}
err := mutator.SelectDefaultTarget().Apply(context.Background(), bundle)
assert.ErrorContains(t, err, "no targets defined")
}
func TestSelectDefaultTargetSingleTargets(t *testing.T) {
bundle := &bundle.Bundle{
Config: config.Root{
Targets: map[string]*config.Target{
"foo": {},
},
},
}
err := mutator.SelectDefaultTarget().Apply(context.Background(), bundle)
assert.NoError(t, err)
assert.Equal(t, "foo", bundle.Config.Bundle.Target)
}
func TestSelectDefaultTargetNoDefaults(t *testing.T) {
bundle := &bundle.Bundle{
Config: config.Root{
Targets: map[string]*config.Target{
"foo": {},
"bar": {},
"qux": {},
},
},
}
err := mutator.SelectDefaultTarget().Apply(context.Background(), bundle)
assert.ErrorContains(t, err, "please specify target")
}
func TestSelectDefaultTargetNoDefaultsWithNil(t *testing.T) {
bundle := &bundle.Bundle{
Config: config.Root{
Targets: map[string]*config.Target{
"foo": nil,
"bar": nil,
},
},
}
err := mutator.SelectDefaultTarget().Apply(context.Background(), bundle)
assert.ErrorContains(t, err, "please specify target")
}
func TestSelectDefaultTargetMultipleDefaults(t *testing.T) {
bundle := &bundle.Bundle{
Config: config.Root{
Targets: map[string]*config.Target{
"foo": {Default: true},
"bar": {Default: true},
"qux": {Default: true},
},
},
}
err := mutator.SelectDefaultTarget().Apply(context.Background(), bundle)
assert.ErrorContains(t, err, "multiple targets are marked as default")
}
func TestSelectDefaultTargetSingleDefault(t *testing.T) {
bundle := &bundle.Bundle{
Config: config.Root{
Targets: map[string]*config.Target{
"foo": {},
"bar": {Default: true},
"qux": {},
},
},
}
err := mutator.SelectDefaultTarget().Apply(context.Background(), bundle)
assert.NoError(t, err)
assert.Equal(t, "bar", bundle.Config.Bundle.Target)
}

View File

@ -1,48 +0,0 @@
package mutator
import (
"context"
"fmt"
"github.com/databricks/cli/bundle"
)
type selectEnvironment struct {
name string
}
// SelectEnvironment merges the specified environment into the root configuration.
func SelectEnvironment(name string) bundle.Mutator {
return &selectEnvironment{
name: name,
}
}
func (m *selectEnvironment) Name() string {
return fmt.Sprintf("SelectEnvironment(%s)", m.name)
}
func (m *selectEnvironment) Apply(_ context.Context, b *bundle.Bundle) error {
if b.Config.Environments == nil {
return fmt.Errorf("no environments defined")
}
// Get specified environment
env, ok := b.Config.Environments[m.name]
if !ok {
return fmt.Errorf("%s: no such environment", m.name)
}
// Merge specified environment into root configuration structure.
err := b.Config.MergeEnvironment(env)
if err != nil {
return err
}
// Store specified environment in configuration for reference.
b.Config.Bundle.Environment = m.name
// Clear environments after loading.
b.Config.Environments = nil
return nil
}

View File

@ -0,0 +1,54 @@
package mutator
import (
"context"
"fmt"
"github.com/databricks/cli/bundle"
)
type selectTarget struct {
name string
}
// SelectTarget merges the specified target into the root configuration.
func SelectTarget(name string) bundle.Mutator {
return &selectTarget{
name: name,
}
}
func (m *selectTarget) Name() string {
return fmt.Sprintf("SelectTarget(%s)", m.name)
}
func (m *selectTarget) Apply(_ context.Context, b *bundle.Bundle) error {
if b.Config.Targets == nil {
return fmt.Errorf("no targets defined")
}
// Get specified target
target, ok := b.Config.Targets[m.name]
if !ok {
return fmt.Errorf("%s: no such target", m.name)
}
// Merge specified target into root configuration structure.
err := b.Config.MergeTargetOverrides(target)
if err != nil {
return err
}
// Store specified target in configuration for reference.
b.Config.Bundle.Target = m.name
// We do this for backward compatibility.
// TODO: remove when Environments section is not supported anymore.
b.Config.Bundle.Environment = b.Config.Bundle.Target
// Clear targets after loading.
b.Config.Targets = nil
b.Config.Environments = nil
return nil
}

View File

@ -11,13 +11,13 @@ import (
"github.com/stretchr/testify/require"
)
func TestSelectEnvironment(t *testing.T) {
func TestSelectTarget(t *testing.T) {
bundle := &bundle.Bundle{
Config: config.Root{
Workspace: config.Workspace{
Host: "foo",
},
Environments: map[string]*config.Environment{
Targets: map[string]*config.Target{
"default": {
Workspace: &config.Workspace{
Host: "bar",
@ -26,19 +26,19 @@ func TestSelectEnvironment(t *testing.T) {
},
},
}
err := mutator.SelectEnvironment("default").Apply(context.Background(), bundle)
err := mutator.SelectTarget("default").Apply(context.Background(), bundle)
require.NoError(t, err)
assert.Equal(t, "bar", bundle.Config.Workspace.Host)
}
func TestSelectEnvironmentNotFound(t *testing.T) {
func TestSelectTargetNotFound(t *testing.T) {
bundle := &bundle.Bundle{
Config: config.Root{
Environments: map[string]*config.Environment{
Targets: map[string]*config.Target{
"default": {},
},
},
}
err := mutator.SelectEnvironment("doesnt-exist").Apply(context.Background(), bundle)
require.Error(t, err, "no environments defined")
err := mutator.SelectTarget("doesnt-exist").Apply(context.Background(), bundle)
require.Error(t, err, "no targets defined")
}

View File

@ -115,7 +115,7 @@ func (r *Resources) SetConfigFilePath(path string) {
}
// MergeJobClusters iterates over all jobs and merges their job clusters.
// This is called after applying the environment overrides.
// This is called after applying the target overrides.
func (r *Resources) MergeJobClusters() error {
for _, job := range r.Jobs {
if err := job.MergeJobClusters(); err != nil {

View File

@ -22,7 +22,7 @@ func (j *Job) MergeJobClusters() error {
keys := make(map[string]*jobs.JobCluster)
output := make([]jobs.JobCluster, 0, len(j.JobClusters))
// Environment overrides are always appended, so we can iterate in natural order to
// Target overrides are always appended, so we can iterate in natural order to
// first find the base definition, and merge instances we encounter later.
for i := range j.JobClusters {
key := j.JobClusters[i].JobClusterKey

View File

@ -69,11 +69,14 @@ type Root struct {
// to deploy in this bundle (e.g. jobs, pipelines, etc.).
Resources Resources `json:"resources,omitempty"`
// Environments can be used to differentiate settings and resources between
// bundle deployment environments (e.g. development, staging, production).
// Targets can be used to differentiate settings and resources between
// bundle deployment targets (e.g. development, staging, production).
// If not specified, the code below initializes this field with a
// single default-initialized environment called "default".
Environments map[string]*Environment `json:"environments,omitempty"`
// single default-initialized target called "default".
Targets map[string]*Target `json:"targets,omitempty"`
// DEPRECATED. Left for backward compatibility with Targets
Environments map[string]*Target `json:"environments,omitempty"`
}
func Load(path string) (*Root, error) {
@ -103,8 +106,8 @@ func Load(path string) (*Root, error) {
// was loaded from in configuration leafs that require it.
func (r *Root) SetConfigFilePath(path string) {
r.Resources.SetConfigFilePath(path)
if r.Environments != nil {
for _, env := range r.Environments {
if r.Targets != nil {
for _, env := range r.Targets {
if env == nil {
continue
}
@ -148,6 +151,15 @@ func (r *Root) Load(path string) error {
return fmt.Errorf("failed to load %s: %w", path, err)
}
if r.Environments != nil && r.Targets != nil {
return fmt.Errorf("both 'environments' and 'targets' are specified, only 'targets' should be used: %s", path)
}
if r.Environments != nil {
//TODO: add a command line notice that this is a deprecated option.
r.Targets = r.Environments
}
r.Path = filepath.Dir(path)
r.SetConfigFilePath(path)
@ -169,37 +181,37 @@ func (r *Root) Merge(other *Root) error {
return mergo.Merge(r, other, mergo.WithOverride)
}
func (r *Root) MergeEnvironment(env *Environment) error {
func (r *Root) MergeTargetOverrides(target *Target) error {
var err error
// Environment may be nil if it's empty.
if env == nil {
// Target may be nil if it's empty.
if target == nil {
return nil
}
if env.Bundle != nil {
err = mergo.Merge(&r.Bundle, env.Bundle, mergo.WithOverride)
if target.Bundle != nil {
err = mergo.Merge(&r.Bundle, target.Bundle, mergo.WithOverride)
if err != nil {
return err
}
}
if env.Workspace != nil {
err = mergo.Merge(&r.Workspace, env.Workspace, mergo.WithOverride)
if target.Workspace != nil {
err = mergo.Merge(&r.Workspace, target.Workspace, mergo.WithOverride)
if err != nil {
return err
}
}
if env.Artifacts != nil {
err = mergo.Merge(&r.Artifacts, env.Artifacts, mergo.WithOverride, mergo.WithAppendSlice)
if target.Artifacts != nil {
err = mergo.Merge(&r.Artifacts, target.Artifacts, mergo.WithOverride, mergo.WithAppendSlice)
if err != nil {
return err
}
}
if env.Resources != nil {
err = mergo.Merge(&r.Resources, env.Resources, mergo.WithOverride, mergo.WithAppendSlice)
if target.Resources != nil {
err = mergo.Merge(&r.Resources, target.Resources, mergo.WithOverride, mergo.WithAppendSlice)
if err != nil {
return err
}
@ -210,8 +222,8 @@ func (r *Root) MergeEnvironment(env *Environment) error {
}
}
if env.Variables != nil {
for k, v := range env.Variables {
if target.Variables != nil {
for k, v := range target.Variables {
variable, ok := r.Variables[k]
if !ok {
return fmt.Errorf("variable %s is not defined but is assigned a value", k)
@ -222,24 +234,24 @@ func (r *Root) MergeEnvironment(env *Environment) error {
}
}
if env.Mode != "" {
r.Bundle.Mode = env.Mode
if target.Mode != "" {
r.Bundle.Mode = target.Mode
}
if env.ComputeID != "" {
r.Bundle.ComputeID = env.ComputeID
if target.ComputeID != "" {
r.Bundle.ComputeID = target.ComputeID
}
git := &r.Bundle.Git
if env.Git.Branch != "" {
git.Branch = env.Git.Branch
if target.Git.Branch != "" {
git.Branch = target.Git.Branch
git.Inferred = false
}
if env.Git.Commit != "" {
git.Commit = env.Git.Commit
if target.Git.Commit != "" {
git.Commit = target.Git.Commit
}
if env.Git.OriginURL != "" {
git.OriginURL = env.Git.OriginURL
if target.Git.OriginURL != "" {
git.OriginURL = target.Git.OriginURL
}
return nil

View File

@ -57,7 +57,7 @@ func TestRootMergeStruct(t *testing.T) {
func TestRootMergeMap(t *testing.T) {
root := &Root{
Path: "path",
Environments: map[string]*Environment{
Targets: map[string]*Target{
"development": {
Workspace: &Workspace{
Host: "foo",
@ -68,7 +68,7 @@ func TestRootMergeMap(t *testing.T) {
}
other := &Root{
Path: "path",
Environments: map[string]*Environment{
Targets: map[string]*Target{
"development": {
Workspace: &Workspace{
Host: "bar",
@ -77,7 +77,7 @@ func TestRootMergeMap(t *testing.T) {
},
}
assert.NoError(t, root.Merge(other))
assert.Equal(t, &Workspace{Host: "bar", Profile: "profile"}, root.Environments["development"].Workspace)
assert.Equal(t, &Workspace{Host: "bar", Profile: "profile"}, root.Targets["development"].Workspace)
}
func TestDuplicateIdOnLoadReturnsError(t *testing.T) {
@ -159,12 +159,12 @@ func TestInitializeVariablesUndefinedVariables(t *testing.T) {
assert.ErrorContains(t, err, "variable bar has not been defined")
}
func TestRootMergeEnvironmentWithMode(t *testing.T) {
func TestRootMergeTargetOverridesWithMode(t *testing.T) {
root := &Root{
Bundle: Bundle{},
}
env := &Environment{Mode: Development}
require.NoError(t, root.MergeEnvironment(env))
env := &Target{Mode: Development}
require.NoError(t, root.MergeTargetOverrides(env))
assert.Equal(t, Development, root.Bundle.Mode)
}

View File

@ -2,14 +2,14 @@ package config
type Mode string
// Environment defines overrides for a single environment.
// Target defines overrides for a single target.
// This structure is recursively merged into the root configuration.
type Environment struct {
// Default marks that this environment must be used if one isn't specified
// by the user (through environment variable or command line argument).
type Target struct {
// Default marks that this target must be used if one isn't specified
// by the user (through target variable or command line argument).
Default bool `json:"default,omitempty"`
// Determines the mode of the environment.
// Determines the mode of the target.
// For example, 'mode: development' can be used for deployments for
// development purposes.
Mode Mode `json:"mode,omitempty"`
@ -27,7 +27,7 @@ type Environment struct {
// Override default values for defined variables
// Does not permit defining new variables or redefining existing ones
// in the scope of an environment
// in the scope of an target
Variables map[string]string `json:"variables,omitempty"`
Git Git `json:"git,omitempty"`

View File

@ -18,7 +18,7 @@ type Variable struct {
// resolved in the following priority order (from highest to lowest)
//
// 1. Command line flag. For example: `--var="foo=bar"`
// 2. Environment variable. eg: BUNDLE_VAR_foo=bar
// 2. Target variable. eg: BUNDLE_VAR_foo=bar
// 3. Default value as defined in the applicable environments block
// 4. Default value defined in variable definition
// 5. Throw error, since if no default value is defined, then the variable

View File

@ -45,7 +45,7 @@ type Workspace struct {
CurrentUser *User `json:"current_user,omitempty" bundle:"readonly"`
// Remote workspace base path for deployment state, for artifacts, as synchronization target.
// This defaults to "~/.bundle/${bundle.name}/${bundle.environment}" where "~" expands to
// This defaults to "~/.bundle/${bundle.name}/${bundle.target}" where "~" expands to
// the current user's home directory in the workspace (e.g. `/Users/jane@doe.com`).
RootPath string `json:"root_path,omitempty"`

View File

@ -31,7 +31,7 @@ func TestInitEnvironmentVariables(t *testing.T) {
Config: config.Root{
Path: t.TempDir(),
Bundle: config.Bundle{
Environment: "whatever",
Target: "whatever",
Terraform: &config.Terraform{
ExecPath: "terraform",
},
@ -58,7 +58,7 @@ func TestSetTempDirEnvVarsForUnixWithTmpDirSet(t *testing.T) {
Config: config.Root{
Path: t.TempDir(),
Bundle: config.Bundle{
Environment: "whatever",
Target: "whatever",
},
},
}
@ -86,7 +86,7 @@ func TestSetTempDirEnvVarsForUnixWithTmpDirNotSet(t *testing.T) {
Config: config.Root{
Path: t.TempDir(),
Bundle: config.Bundle{
Environment: "whatever",
Target: "whatever",
},
},
}
@ -112,7 +112,7 @@ func TestSetTempDirEnvVarsForWindowWithAllTmpDirEnvVarsSet(t *testing.T) {
Config: config.Root{
Path: t.TempDir(),
Bundle: config.Bundle{
Environment: "whatever",
Target: "whatever",
},
},
}
@ -142,7 +142,7 @@ func TestSetTempDirEnvVarsForWindowWithUserProfileAndTempSet(t *testing.T) {
Config: config.Root{
Path: t.TempDir(),
Bundle: config.Bundle{
Environment: "whatever",
Target: "whatever",
},
},
}
@ -172,7 +172,7 @@ func TestSetTempDirEnvVarsForWindowWithUserProfileSet(t *testing.T) {
Config: config.Root{
Path: t.TempDir(),
Bundle: config.Bundle{
Environment: "whatever",
Target: "whatever",
},
},
}
@ -202,7 +202,7 @@ func TestSetTempDirEnvVarsForWindowsWithoutAnyTempDirEnvVarsSet(t *testing.T) {
Config: config.Root{
Path: t.TempDir(),
Bundle: config.Bundle{
Environment: "whatever",
Target: "whatever",
},
},
}
@ -230,7 +230,7 @@ func TestSetProxyEnvVars(t *testing.T) {
Config: config.Root{
Path: t.TempDir(),
Bundle: config.Bundle{
Environment: "whatever",
Target: "whatever",
},
},
}

View File

@ -20,7 +20,7 @@ func TestLoadWithNoState(t *testing.T) {
Config: config.Root{
Path: t.TempDir(),
Bundle: config.Bundle{
Environment: "whatever",
Target: "whatever",
Terraform: &config.Terraform{
ExecPath: "terraform",
},

View File

@ -26,7 +26,7 @@ func Initialize() bundle.Mutator {
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
),
mutator.OverrideCompute(),
mutator.ProcessEnvironmentMode(),
mutator.ProcessTargetMode(),
mutator.TranslatePaths(),
terraform.Initialize(),
},

View File

@ -3,7 +3,7 @@
`docs/bundle_descriptions.json` contains both autogenerated as well as manually written
descriptions for the json schema. Specifically
1. `resources` : almost all descriptions are autogenerated from the OpenAPI spec
2. `environments` : almost all descriptions are copied over from root level entities (eg: `bundle`, `artifacts`)
2. `targets` : almost all descriptions are copied over from root level entities (eg: `bundle`, `artifacts`)
3. `bundle` : manually editted
4. `include` : manually editted
5. `workspace` : manually editted
@ -17,7 +17,7 @@ These descriptions are rendered in the inline documentation in an IDE
`databricks bundle schema --only-docs > ~/databricks/bundle/schema/docs/bundle_descriptions.json`
2. Manually edit bundle_descriptions.json to add your descriptions
3. Build again to embed the new `bundle_descriptions.json` into the binary (`go build`)
4. Again run `databricks bundle schema --only-docs > ~/databricks/bundle/schema/docs/bundle_descriptions.json` to copy over any applicable descriptions to `environments`
4. Again run `databricks bundle schema --only-docs > ~/databricks/bundle/schema/docs/bundle_descriptions.json` to copy over any applicable descriptions to `targets`
5. push to repo

View File

@ -52,20 +52,20 @@ func BundleDocs(openapiSpecPath string) (*Docs, error) {
}
docs.Properties["resources"] = schemaToDocs(resourceSchema)
}
docs.refreshEnvironmentsDocs()
docs.refreshTargetsDocs()
return docs, nil
}
func (docs *Docs) refreshEnvironmentsDocs() error {
environmentsDocs, ok := docs.Properties["environments"]
if !ok || environmentsDocs.AdditionalProperties == nil ||
environmentsDocs.AdditionalProperties.Properties == nil {
return fmt.Errorf("invalid environments descriptions")
func (docs *Docs) refreshTargetsDocs() error {
targetsDocs, ok := docs.Properties["targets"]
if !ok || targetsDocs.AdditionalProperties == nil ||
targetsDocs.AdditionalProperties.Properties == nil {
return fmt.Errorf("invalid targets descriptions")
}
environmentProperties := environmentsDocs.AdditionalProperties.Properties
targetProperties := targetsDocs.AdditionalProperties.Properties
propertiesToCopy := []string{"artifacts", "bundle", "resources", "workspace"}
for _, p := range propertiesToCopy {
environmentProperties[p] = docs.Properties[p]
targetProperties[p] = docs.Properties[p]
}
return nil
}

View File

@ -36,7 +36,7 @@
}
}
},
"environments": {
"targets": {
"description": "",
"additionalproperties": {
"description": "",
@ -1827,7 +1827,7 @@
"description": "Connection profile to use. By default profiles are specified in ~/.databrickscfg."
},
"root_path": {
"description": "The base location for synchronizing files, artifacts and state. Defaults to `/Users/jane@doe.com/.bundle/${bundle.name}/${bundle.environment}`"
"description": "The base location for synchronizing files, artifacts and state. Defaults to `/Users/jane@doe.com/.bundle/${bundle.name}/${bundle.target}`"
},
"state_path": {
"description": "The remote path to synchronize bundle state to. This defaults to `${workspace.root}/state`"
@ -3591,7 +3591,7 @@
"description": "Connection profile to use. By default profiles are specified in ~/.databrickscfg."
},
"root_path": {
"description": "The base location for synchronizing files, artifacts and state. Defaults to `/Users/jane@doe.com/.bundle/${bundle.name}/${bundle.environment}`"
"description": "The base location for synchronizing files, artifacts and state. Defaults to `/Users/jane@doe.com/.bundle/${bundle.name}/${bundle.target}`"
},
"state_path": {
"description": "The remote path to synchronize bundle state to. This defaults to `${workspace.root}/state`"

View File

@ -1,7 +1,7 @@
bundle:
name: autoload git config test
environments:
targets:
development:
default: true

View File

@ -1,5 +0,0 @@
bundle:
name: environment_empty
environments:
development:

View File

@ -1,12 +0,0 @@
package config_tests
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestEnvironmentEmpty(t *testing.T) {
b := loadEnvironment(t, "./environment_empty", "development")
assert.Equal(t, "development", b.Config.Bundle.Environment)
}

View File

@ -0,0 +1,20 @@
package config_tests
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestGitAutoLoadWithEnvironment(t *testing.T) {
b := load(t, "./environments_autoload_git")
assert.True(t, b.Config.Bundle.Git.Inferred)
assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli")
}
func TestGitManuallySetBranchWithEnvironment(t *testing.T) {
b := loadTarget(t, "./environments_autoload_git", "production")
assert.False(t, b.Config.Bundle.Git.Inferred)
assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli")
}

View File

@ -7,17 +7,17 @@ import (
)
func TestEnvironmentOverridesWorkspaceDev(t *testing.T) {
b := loadEnvironment(t, "./environment_overrides/workspace", "development")
b := loadTarget(t, "./environment_overrides/workspace", "development")
assert.Equal(t, "https://development.acme.cloud.databricks.com/", b.Config.Workspace.Host)
}
func TestEnvironmentOverridesWorkspaceStaging(t *testing.T) {
b := loadEnvironment(t, "./environment_overrides/workspace", "staging")
b := loadTarget(t, "./environment_overrides/workspace", "staging")
assert.Equal(t, "https://staging.acme.cloud.databricks.com/", b.Config.Workspace.Host)
}
func TestEnvironmentOverridesResourcesDev(t *testing.T) {
b := loadEnvironment(t, "./environment_overrides/resources", "development")
b := loadTarget(t, "./environment_overrides/resources", "development")
assert.Equal(t, "base job", b.Config.Resources.Jobs["job1"].Name)
// Base values are preserved in the development environment.
@ -26,7 +26,7 @@ func TestEnvironmentOverridesResourcesDev(t *testing.T) {
}
func TestEnvironmentOverridesResourcesStaging(t *testing.T) {
b := loadEnvironment(t, "./environment_overrides/resources", "staging")
b := loadTarget(t, "./environment_overrides/resources", "staging")
assert.Equal(t, "staging job", b.Config.Resources.Jobs["job1"].Name)
// Overrides are only applied if they are not zero-valued.

View File

@ -0,0 +1,11 @@
bundle:
name: autoload git config test
environments:
development:
default: true
production:
# production can only be deployed from the 'main' branch
git:
branch: main

View File

@ -0,0 +1,44 @@
resources:
pipelines:
nyc_taxi_pipeline:
name: "nyc taxi loader"
libraries:
- notebook:
path: ./dlt/nyc_taxi_loader
environments:
development:
mode: development
resources:
pipelines:
nyc_taxi_pipeline:
target: nyc_taxi_development
development: true
staging:
resources:
pipelines:
nyc_taxi_pipeline:
target: nyc_taxi_staging
development: false
production:
mode: production
resources:
pipelines:
nyc_taxi_pipeline:
target: nyc_taxi_production
development: false
photon: true
jobs:
pipeline_schedule:
name: Daily refresh of production pipeline
schedule:
quartz_cron_expression: 6 6 11 * * ?
timezone_id: UTC
tasks:
- pipeline_task:
pipeline_id: "to be interpolated"

View File

@ -0,0 +1,56 @@
package config_tests
import (
"path/filepath"
"testing"
"github.com/databricks/cli/bundle/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestJobAndPipelineDevelopmentWithEnvironment(t *testing.T) {
b := loadTarget(t, "./environments_job_and_pipeline", "development")
assert.Len(t, b.Config.Resources.Jobs, 0)
assert.Len(t, b.Config.Resources.Pipelines, 1)
p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"]
assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath))
assert.Equal(t, b.Config.Bundle.Mode, config.Development)
assert.True(t, p.Development)
require.Len(t, p.Libraries, 1)
assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path)
assert.Equal(t, "nyc_taxi_development", p.Target)
}
func TestJobAndPipelineStagingWithEnvironment(t *testing.T) {
b := loadTarget(t, "./environments_job_and_pipeline", "staging")
assert.Len(t, b.Config.Resources.Jobs, 0)
assert.Len(t, b.Config.Resources.Pipelines, 1)
p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"]
assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath))
assert.False(t, p.Development)
require.Len(t, p.Libraries, 1)
assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path)
assert.Equal(t, "nyc_taxi_staging", p.Target)
}
func TestJobAndPipelineProductionWithEnvironment(t *testing.T) {
b := loadTarget(t, "./environments_job_and_pipeline", "production")
assert.Len(t, b.Config.Resources.Jobs, 1)
assert.Len(t, b.Config.Resources.Pipelines, 1)
p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"]
assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath))
assert.False(t, p.Development)
require.Len(t, p.Libraries, 1)
assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path)
assert.Equal(t, "nyc_taxi_production", p.Target)
j := b.Config.Resources.Jobs["pipeline_schedule"]
assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(j.ConfigFilePath))
assert.Equal(t, "Daily refresh of production pipeline", j.Name)
require.Len(t, j.Tasks, 1)
assert.NotEmpty(t, j.Tasks[0].PipelineTask.PipelineId)
}

View File

@ -0,0 +1,35 @@
bundle:
name: override_job_cluster
workspace:
host: https://acme.cloud.databricks.com/
resources:
jobs:
foo:
name: job
job_clusters:
- job_cluster_key: key
new_cluster:
spark_version: 13.3.x-scala2.12
environments:
development:
resources:
jobs:
foo:
job_clusters:
- job_cluster_key: key
new_cluster:
node_type_id: i3.xlarge
num_workers: 1
staging:
resources:
jobs:
foo:
job_clusters:
- job_cluster_key: key
new_cluster:
node_type_id: i3.2xlarge
num_workers: 4

View File

@ -0,0 +1,29 @@
package config_tests
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestOverrideJobClusterDevWithEnvironment(t *testing.T) {
b := loadTarget(t, "./environments_override_job_cluster", "development")
assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name)
assert.Len(t, b.Config.Resources.Jobs["foo"].JobClusters, 1)
c := b.Config.Resources.Jobs["foo"].JobClusters[0]
assert.Equal(t, "13.3.x-scala2.12", c.NewCluster.SparkVersion)
assert.Equal(t, "i3.xlarge", c.NewCluster.NodeTypeId)
assert.Equal(t, 1, c.NewCluster.NumWorkers)
}
func TestOverrideJobClusterStagingWithEnvironment(t *testing.T) {
b := loadTarget(t, "./environments_override_job_cluster", "staging")
assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name)
assert.Len(t, b.Config.Resources.Jobs["foo"].JobClusters, 1)
c := b.Config.Resources.Jobs["foo"].JobClusters[0]
assert.Equal(t, "13.3.x-scala2.12", c.NewCluster.SparkVersion)
assert.Equal(t, "i3.2xlarge", c.NewCluster.NodeTypeId)
assert.Equal(t, 4, c.NewCluster.NumWorkers)
}

View File

@ -17,7 +17,7 @@ func TestGitAutoLoad(t *testing.T) {
}
func TestGitManuallySetBranch(t *testing.T) {
b := loadEnvironment(t, "./autoload_git", "production")
b := loadTarget(t, "./autoload_git", "production")
assert.False(t, b.Config.Bundle.Git.Inferred)
assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli")

View File

@ -0,0 +1,14 @@
bundle:
name: foo ${workspace.profile}
workspace:
profile: bar
targets:
development:
default: true
resources:
jobs:
my_job:
name: "${bundle.name} | ${workspace.profile} | ${bundle.environment} | ${bundle.target}"

View File

@ -20,3 +20,15 @@ func TestInterpolation(t *testing.T) {
assert.Equal(t, "foo bar", b.Config.Bundle.Name)
assert.Equal(t, "foo bar | bar", b.Config.Resources.Jobs["my_job"].Name)
}
func TestInterpolationWithTarget(t *testing.T) {
b := loadTarget(t, "./interpolation_target", "development")
err := bundle.Apply(context.Background(), b, interpolation.Interpolate(
interpolation.IncludeLookupsInPath("bundle"),
interpolation.IncludeLookupsInPath("workspace"),
))
require.NoError(t, err)
assert.Equal(t, "foo bar", b.Config.Bundle.Name)
assert.Equal(t, "foo bar | bar | development | development", b.Config.Resources.Jobs["my_job"].Name)
}

View File

@ -6,7 +6,7 @@ resources:
- notebook:
path: ./dlt/nyc_taxi_loader
environments:
targets:
development:
mode: development
resources:

View File

@ -10,7 +10,7 @@ import (
)
func TestJobAndPipelineDevelopment(t *testing.T) {
b := loadEnvironment(t, "./job_and_pipeline", "development")
b := loadTarget(t, "./job_and_pipeline", "development")
assert.Len(t, b.Config.Resources.Jobs, 0)
assert.Len(t, b.Config.Resources.Pipelines, 1)
@ -24,7 +24,7 @@ func TestJobAndPipelineDevelopment(t *testing.T) {
}
func TestJobAndPipelineStaging(t *testing.T) {
b := loadEnvironment(t, "./job_and_pipeline", "staging")
b := loadTarget(t, "./job_and_pipeline", "staging")
assert.Len(t, b.Config.Resources.Jobs, 0)
assert.Len(t, b.Config.Resources.Pipelines, 1)
@ -37,7 +37,7 @@ func TestJobAndPipelineStaging(t *testing.T) {
}
func TestJobAndPipelineProduction(t *testing.T) {
b := loadEnvironment(t, "./job_and_pipeline", "production")
b := loadTarget(t, "./job_and_pipeline", "production")
assert.Len(t, b.Config.Resources.Jobs, 1)
assert.Len(t, b.Config.Resources.Pipelines, 1)

View File

@ -18,9 +18,9 @@ func load(t *testing.T, path string) *bundle.Bundle {
return b
}
func loadEnvironment(t *testing.T, path, env string) *bundle.Bundle {
func loadTarget(t *testing.T, path, env string) *bundle.Bundle {
b := load(t, path)
err := bundle.Apply(context.Background(), b, mutator.SelectEnvironment(env))
err := bundle.Apply(context.Background(), b, mutator.SelectTarget(env))
require.NoError(t, err)
return b
}

View File

@ -13,7 +13,7 @@ resources:
new_cluster:
spark_version: 13.3.x-scala2.12
environments:
targets:
development:
resources:
jobs:

View File

@ -7,7 +7,7 @@ import (
)
func TestOverrideJobClusterDev(t *testing.T) {
b := loadEnvironment(t, "./override_job_cluster", "development")
b := loadTarget(t, "./override_job_cluster", "development")
assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name)
assert.Len(t, b.Config.Resources.Jobs["foo"].JobClusters, 1)
@ -18,7 +18,7 @@ func TestOverrideJobClusterDev(t *testing.T) {
}
func TestOverrideJobClusterStaging(t *testing.T) {
b := loadEnvironment(t, "./override_job_cluster", "staging")
b := loadTarget(t, "./override_job_cluster", "staging")
assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name)
assert.Len(t, b.Config.Resources.Jobs["foo"].JobClusters, 1)

View File

@ -0,0 +1,5 @@
bundle:
name: target_empty
targets:
development:

View File

@ -0,0 +1,12 @@
package config_tests
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestTargetEmpty(t *testing.T) {
b := loadTarget(t, "./target_empty", "development")
assert.Equal(t, "development", b.Config.Bundle.Target)
}

View File

@ -0,0 +1,20 @@
bundle:
name: environment_overrides
workspace:
host: https://acme.cloud.databricks.com/
resources:
jobs:
job1:
name: "base job"
targets:
development:
default: true
staging:
resources:
jobs:
job1:
name: "staging job"

View File

@ -0,0 +1,14 @@
bundle:
name: environment_overrides
workspace:
host: https://acme.cloud.databricks.com/
targets:
development:
workspace:
host: https://development.acme.cloud.databricks.com/
staging:
workspace:
host: https://staging.acme.cloud.databricks.com/

View File

@ -0,0 +1,27 @@
package config_tests
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestTargetOverridesWorkspaceDev(t *testing.T) {
b := loadTarget(t, "./target_overrides/workspace", "development")
assert.Equal(t, "https://development.acme.cloud.databricks.com/", b.Config.Workspace.Host)
}
func TestTargetOverridesWorkspaceStaging(t *testing.T) {
b := loadTarget(t, "./target_overrides/workspace", "staging")
assert.Equal(t, "https://staging.acme.cloud.databricks.com/", b.Config.Workspace.Host)
}
func TestTargetOverridesResourcesDev(t *testing.T) {
b := loadTarget(t, "./target_overrides/resources", "development")
assert.Equal(t, "base job", b.Config.Resources.Jobs["job1"].Name)
}
func TestTargetOverridesResourcesStaging(t *testing.T) {
b := loadTarget(t, "./target_overrides/resources", "staging")
assert.Equal(t, "staging job", b.Config.Resources.Jobs["job1"].Name)
}

View File

@ -12,7 +12,7 @@ bundle:
workspace:
profile: ${var.a} ${var.b}
environments:
targets:
env-with-single-variable-override:
variables:
b: dev-b

View File

@ -34,10 +34,10 @@ func TestVariablesLoadingFailsWhenRequiredVariableIsNotSpecified(t *testing.T) {
assert.ErrorContains(t, err, "no value assigned to required variable b. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_b environment variable")
}
func TestVariablesEnvironmentsBlockOverride(t *testing.T) {
func TestVariablesTargetsBlockOverride(t *testing.T) {
b := load(t, "./variables/env_overrides")
err := bundle.Apply(context.Background(), b, bundle.Seq(
mutator.SelectEnvironment("env-with-single-variable-override"),
mutator.SelectTarget("env-with-single-variable-override"),
mutator.SetVariables(),
interpolation.Interpolate(
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
@ -46,10 +46,10 @@ func TestVariablesEnvironmentsBlockOverride(t *testing.T) {
assert.Equal(t, "default-a dev-b", b.Config.Workspace.Profile)
}
func TestVariablesEnvironmentsBlockOverrideForMultipleVariables(t *testing.T) {
func TestVariablesTargetsBlockOverrideForMultipleVariables(t *testing.T) {
b := load(t, "./variables/env_overrides")
err := bundle.Apply(context.Background(), b, bundle.Seq(
mutator.SelectEnvironment("env-with-two-variable-overrides"),
mutator.SelectTarget("env-with-two-variable-overrides"),
mutator.SetVariables(),
interpolation.Interpolate(
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
@ -58,11 +58,11 @@ func TestVariablesEnvironmentsBlockOverrideForMultipleVariables(t *testing.T) {
assert.Equal(t, "prod-a prod-b", b.Config.Workspace.Profile)
}
func TestVariablesEnvironmentsBlockOverrideWithProcessEnvVars(t *testing.T) {
func TestVariablesTargetsBlockOverrideWithProcessEnvVars(t *testing.T) {
t.Setenv("BUNDLE_VAR_b", "env-var-b")
b := load(t, "./variables/env_overrides")
err := bundle.Apply(context.Background(), b, bundle.Seq(
mutator.SelectEnvironment("env-with-two-variable-overrides"),
mutator.SelectTarget("env-with-two-variable-overrides"),
mutator.SetVariables(),
interpolation.Interpolate(
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
@ -71,10 +71,10 @@ func TestVariablesEnvironmentsBlockOverrideWithProcessEnvVars(t *testing.T) {
assert.Equal(t, "prod-a env-var-b", b.Config.Workspace.Profile)
}
func TestVariablesEnvironmentsBlockOverrideWithMissingVariables(t *testing.T) {
func TestVariablesTargetsBlockOverrideWithMissingVariables(t *testing.T) {
b := load(t, "./variables/env_overrides")
err := bundle.Apply(context.Background(), b, bundle.Seq(
mutator.SelectEnvironment("env-missing-a-required-variable-assignment"),
mutator.SelectTarget("env-missing-a-required-variable-assignment"),
mutator.SetVariables(),
interpolation.Interpolate(
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
@ -82,10 +82,10 @@ func TestVariablesEnvironmentsBlockOverrideWithMissingVariables(t *testing.T) {
assert.ErrorContains(t, err, "no value assigned to required variable b. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_b environment variable")
}
func TestVariablesEnvironmentsBlockOverrideWithUndefinedVariables(t *testing.T) {
func TestVariablesTargetsBlockOverrideWithUndefinedVariables(t *testing.T) {
b := load(t, "./variables/env_overrides")
err := bundle.Apply(context.Background(), b, bundle.Seq(
mutator.SelectEnvironment("env-using-an-undefined-variable"),
mutator.SelectTarget("env-using-an-undefined-variable"),
mutator.SetVariables(),
interpolation.Interpolate(
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),

View File

@ -7,7 +7,7 @@ import (
)
func ConfigureBundleWithVariables(cmd *cobra.Command, args []string) error {
// Load bundle config and apply environment
// Load bundle config and apply target
err := root.MustConfigureBundle(cmd, args)
if err != nil {
return err

View File

@ -131,7 +131,7 @@ func newConfigureCommand() *cobra.Command {
// Include token flag for compatibility with the legacy CLI.
// It doesn't actually do anything because we always use PATs.
cmd.Flags().BoolP("token", "t", true, "Configure using Databricks Personal Access Token")
cmd.Flags().Bool("token", true, "Configure using Databricks Personal Access Token")
cmd.Flags().MarkHidden("token")
cmd.RunE = func(cmd *cobra.Command, args []string) error {

View File

@ -11,11 +11,12 @@ import (
)
const envName = "DATABRICKS_BUNDLE_ENV"
const targetName = "DATABRICKS_BUNDLE_TARGET"
// getEnvironment returns the name of the environment to operate in.
func getEnvironment(cmd *cobra.Command) (value string) {
// getTarget returns the name of the target to operate in.
func getTarget(cmd *cobra.Command) (value string) {
// The command line flag takes precedence.
flag := cmd.Flag("environment")
flag := cmd.Flag("target")
if flag != nil {
value = flag.Value.String()
if value != "" {
@ -23,8 +24,23 @@ func getEnvironment(cmd *cobra.Command) (value string) {
}
}
oldFlag := cmd.Flag("environment")
if oldFlag != nil {
value = flag.Value.String()
if value != "" {
return
}
}
// If it's not set, use the environment variable.
return os.Getenv(envName)
target := os.Getenv(targetName)
// If target env is not set with a new variable, try to check for old variable name
// TODO: remove when environments section is not supported anymore
if target == "" {
target = os.Getenv(envName)
}
return target
}
func getProfile(cmd *cobra.Command) (value string) {
@ -80,11 +96,11 @@ func configureBundle(cmd *cobra.Command, args []string, load func(ctx context.Co
}
var m bundle.Mutator
env := getEnvironment(cmd)
env := getTarget(cmd)
if env == "" {
m = mutator.SelectDefaultEnvironment()
m = mutator.SelectDefaultTarget()
} else {
m = mutator.SelectEnvironment(env)
m = mutator.SelectTarget(env)
}
ctx := cmd.Context()
@ -108,19 +124,27 @@ func TryConfigureBundle(cmd *cobra.Command, args []string) error {
return configureBundle(cmd, args, bundle.TryLoad)
}
// environmentCompletion executes to autocomplete the argument to the environment flag.
func environmentCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
// targetCompletion executes to autocomplete the argument to the target flag.
func targetCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
b, err := loadBundle(cmd, args, bundle.MustLoad)
if err != nil {
cobra.CompErrorln(err.Error())
return nil, cobra.ShellCompDirectiveError
}
return maps.Keys(b.Config.Environments), cobra.ShellCompDirectiveDefault
return maps.Keys(b.Config.Targets), cobra.ShellCompDirectiveDefault
}
func initTargetFlag(cmd *cobra.Command) {
// To operate in the context of a bundle, all commands must take an "target" parameter.
cmd.PersistentFlags().StringP("target", "t", "", "bundle target to use (if applicable)")
cmd.RegisterFlagCompletionFunc("target", targetCompletion)
}
// DEPRECATED flag
func initEnvironmentFlag(cmd *cobra.Command) {
// To operate in the context of a bundle, all commands must take an "environment" parameter.
cmd.PersistentFlags().StringP("environment", "e", "", "bundle environment to use (if applicable)")
cmd.RegisterFlagCompletionFunc("environment", environmentCompletion)
cmd.PersistentFlags().StringP("environment", "e", "", "bundle target to use (if applicable)")
cmd.PersistentFlags().MarkDeprecated("environment", "use --target flag instead")
cmd.RegisterFlagCompletionFunc("environment", targetCompletion)
}

View File

@ -128,3 +128,27 @@ func TestBundleConfigureWithProfileFlagAndEnvVariable(t *testing.T) {
b.WorkspaceClient()
})
}
func TestTargetFlagFull(t *testing.T) {
cmd := emptyCommand(t)
initTargetFlag(cmd)
cmd.SetArgs([]string{"version", "--target", "development"})
ctx := context.Background()
err := cmd.ExecuteContext(ctx)
assert.NoError(t, err)
assert.Equal(t, cmd.Flag("target").Value.String(), "development")
}
func TestTargetFlagShort(t *testing.T) {
cmd := emptyCommand(t)
initTargetFlag(cmd)
cmd.SetArgs([]string{"version", "-t", "production"})
ctx := context.Background()
err := cmd.ExecuteContext(ctx)
assert.NoError(t, err)
assert.Equal(t, cmd.Flag("target").Value.String(), "production")
}

View File

@ -36,6 +36,7 @@ func New() *cobra.Command {
outputFlag := initOutputFlag(cmd)
initProfileFlag(cmd)
initEnvironmentFlag(cmd)
initTargetFlag(cmd)
cmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()

View File

@ -18,7 +18,7 @@ func TestSyncOptionsFromBundle(t *testing.T) {
Path: tempDir,
Bundle: config.Bundle{
Environment: "default",
Target: "default",
},
Workspace: config.Workspace{