mirror of https://github.com/databricks/cli.git
Renamed `environments` to `targets` in bundle configuration (#670)
## Changes Renamed Environments to Targets in bundle.yml. The change is backward-compatible and customers can continue to use `environments` in the time being. ## Tests Added tests which checks that both `environments` and `targets` sections in bundle.yml works correctly
This commit is contained in:
parent
4694832534
commit
56dcd3f0a7
|
@ -117,10 +117,10 @@ func (b *Bundle) WorkspaceClient() *databricks.WorkspaceClient {
|
|||
}
|
||||
|
||||
// CacheDir returns directory to use for temporary files for this bundle.
|
||||
// Scoped to the bundle's environment.
|
||||
// Scoped to the bundle's target.
|
||||
func (b *Bundle) CacheDir(paths ...string) (string, error) {
|
||||
if b.Config.Bundle.Environment == "" {
|
||||
panic("environment not set")
|
||||
if b.Config.Bundle.Target == "" {
|
||||
panic("target not set")
|
||||
}
|
||||
|
||||
cacheDirName, exists := os.LookupEnv("DATABRICKS_BUNDLE_TMP")
|
||||
|
@ -138,8 +138,8 @@ func (b *Bundle) CacheDir(paths ...string) (string, error) {
|
|||
// Fixed components of the result path.
|
||||
parts := []string{
|
||||
cacheDirName,
|
||||
// Scope with environment name.
|
||||
b.Config.Bundle.Environment,
|
||||
// Scope with target name.
|
||||
b.Config.Bundle.Target,
|
||||
}
|
||||
|
||||
// Append dynamic components of the result path.
|
||||
|
|
|
@ -31,16 +31,16 @@ func TestBundleCacheDir(t *testing.T) {
|
|||
bundle, err := Load(context.Background(), projectDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Artificially set environment.
|
||||
// This is otherwise done by [mutators.SelectEnvironment].
|
||||
bundle.Config.Bundle.Environment = "default"
|
||||
// Artificially set target.
|
||||
// This is otherwise done by [mutators.SelectTarget].
|
||||
bundle.Config.Bundle.Target = "default"
|
||||
|
||||
// unset env variable in case it's set
|
||||
t.Setenv("DATABRICKS_BUNDLE_TMP", "")
|
||||
|
||||
cacheDir, err := bundle.CacheDir()
|
||||
|
||||
// format is <CWD>/.databricks/bundle/<environment>
|
||||
// format is <CWD>/.databricks/bundle/<target>
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, filepath.Join(projectDir, ".databricks", "bundle", "default"), cacheDir)
|
||||
}
|
||||
|
@ -55,16 +55,16 @@ func TestBundleCacheDirOverride(t *testing.T) {
|
|||
bundle, err := Load(context.Background(), projectDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Artificially set environment.
|
||||
// This is otherwise done by [mutators.SelectEnvironment].
|
||||
bundle.Config.Bundle.Environment = "default"
|
||||
// Artificially set target.
|
||||
// This is otherwise done by [mutators.SelectTarget].
|
||||
bundle.Config.Bundle.Target = "default"
|
||||
|
||||
// now we expect to use 'bundleTmpDir' instead of CWD/.databricks/bundle
|
||||
t.Setenv("DATABRICKS_BUNDLE_TMP", bundleTmpDir)
|
||||
|
||||
cacheDir, err := bundle.CacheDir()
|
||||
|
||||
// format is <DATABRICKS_BUNDLE_TMP>/<environment>
|
||||
// format is <DATABRICKS_BUNDLE_TMP>/<target>
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, filepath.Join(bundleTmpDir, "default"), cacheDir)
|
||||
}
|
||||
|
|
|
@ -15,7 +15,10 @@ type Bundle struct {
|
|||
// Default warehouse to run SQL on.
|
||||
// DefaultWarehouse string `json:"default_warehouse,omitempty"`
|
||||
|
||||
// Environment is set by the mutator that selects the environment.
|
||||
// Target is set by the mutator that selects the target.
|
||||
Target string `json:"target,omitempty" bundle:"readonly"`
|
||||
|
||||
// DEPRECATED. Left for backward compatibility with Target
|
||||
Environment string `json:"environment,omitempty" bundle:"readonly"`
|
||||
|
||||
// Terraform holds configuration related to Terraform.
|
||||
|
@ -32,10 +35,10 @@ type Bundle struct {
|
|||
// origin url. Automatically loaded by reading .git directory if not specified
|
||||
Git Git `json:"git,omitempty"`
|
||||
|
||||
// Determines the mode of the environment.
|
||||
// Determines the mode of the target.
|
||||
// For example, 'mode: development' can be used for deployments for
|
||||
// development purposes.
|
||||
// Annotated readonly as this should be set at the environment level.
|
||||
// Annotated readonly as this should be set at the target level.
|
||||
Mode Mode `json:"mode,omitempty" bundle:"readonly"`
|
||||
|
||||
// Overrides the compute used for jobs and other supported assets.
|
||||
|
|
|
@ -1,37 +0,0 @@
|
|||
package mutator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"github.com/databricks/cli/bundle/config"
|
||||
)
|
||||
|
||||
type defineDefaultEnvironment struct {
|
||||
name string
|
||||
}
|
||||
|
||||
// DefineDefaultEnvironment adds an environment named "default"
|
||||
// to the configuration if none have been defined.
|
||||
func DefineDefaultEnvironment() bundle.Mutator {
|
||||
return &defineDefaultEnvironment{
|
||||
name: "default",
|
||||
}
|
||||
}
|
||||
|
||||
func (m *defineDefaultEnvironment) Name() string {
|
||||
return fmt.Sprintf("DefineDefaultEnvironment(%s)", m.name)
|
||||
}
|
||||
|
||||
func (m *defineDefaultEnvironment) Apply(_ context.Context, b *bundle.Bundle) error {
|
||||
// Nothing to do if the configuration has at least 1 environment.
|
||||
if len(b.Config.Environments) > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Define default environment.
|
||||
b.Config.Environments = make(map[string]*config.Environment)
|
||||
b.Config.Environments[m.name] = &config.Environment{}
|
||||
return nil
|
||||
}
|
|
@ -0,0 +1,37 @@
|
|||
package mutator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"github.com/databricks/cli/bundle/config"
|
||||
)
|
||||
|
||||
type defineDefaultTarget struct {
|
||||
name string
|
||||
}
|
||||
|
||||
// DefineDefaultTarget adds a target named "default"
|
||||
// to the configuration if none have been defined.
|
||||
func DefineDefaultTarget() bundle.Mutator {
|
||||
return &defineDefaultTarget{
|
||||
name: "default",
|
||||
}
|
||||
}
|
||||
|
||||
func (m *defineDefaultTarget) Name() string {
|
||||
return fmt.Sprintf("DefineDefaultTarget(%s)", m.name)
|
||||
}
|
||||
|
||||
func (m *defineDefaultTarget) Apply(_ context.Context, b *bundle.Bundle) error {
|
||||
// Nothing to do if the configuration has at least 1 target.
|
||||
if len(b.Config.Targets) > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Define default target.
|
||||
b.Config.Targets = make(map[string]*config.Target)
|
||||
b.Config.Targets[m.name] = &config.Target{}
|
||||
return nil
|
||||
}
|
|
@ -11,25 +11,25 @@ import (
|
|||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDefaultEnvironment(t *testing.T) {
|
||||
func TestDefaultTarget(t *testing.T) {
|
||||
bundle := &bundle.Bundle{}
|
||||
err := mutator.DefineDefaultEnvironment().Apply(context.Background(), bundle)
|
||||
err := mutator.DefineDefaultTarget().Apply(context.Background(), bundle)
|
||||
require.NoError(t, err)
|
||||
env, ok := bundle.Config.Environments["default"]
|
||||
env, ok := bundle.Config.Targets["default"]
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, &config.Environment{}, env)
|
||||
assert.Equal(t, &config.Target{}, env)
|
||||
}
|
||||
|
||||
func TestDefaultEnvironmentAlreadySpecified(t *testing.T) {
|
||||
func TestDefaultTargetAlreadySpecified(t *testing.T) {
|
||||
bundle := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Environments: map[string]*config.Environment{
|
||||
Targets: map[string]*config.Target{
|
||||
"development": {},
|
||||
},
|
||||
},
|
||||
}
|
||||
err := mutator.DefineDefaultEnvironment().Apply(context.Background(), bundle)
|
||||
err := mutator.DefineDefaultTarget().Apply(context.Background(), bundle)
|
||||
require.NoError(t, err)
|
||||
_, ok := bundle.Config.Environments["default"]
|
||||
_, ok := bundle.Config.Targets["default"]
|
||||
assert.False(t, ok)
|
||||
}
|
|
@ -27,14 +27,14 @@ func (m *defineDefaultWorkspaceRoot) Apply(ctx context.Context, b *bundle.Bundle
|
|||
return fmt.Errorf("unable to define default workspace root: bundle name not defined")
|
||||
}
|
||||
|
||||
if b.Config.Bundle.Environment == "" {
|
||||
return fmt.Errorf("unable to define default workspace root: bundle environment not selected")
|
||||
if b.Config.Bundle.Target == "" {
|
||||
return fmt.Errorf("unable to define default workspace root: bundle target not selected")
|
||||
}
|
||||
|
||||
b.Config.Workspace.RootPath = fmt.Sprintf(
|
||||
"~/.bundle/%s/%s",
|
||||
b.Config.Bundle.Name,
|
||||
b.Config.Bundle.Environment,
|
||||
b.Config.Bundle.Target,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -15,8 +15,8 @@ func TestDefaultWorkspaceRoot(t *testing.T) {
|
|||
bundle := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Name: "name",
|
||||
Environment: "environment",
|
||||
Name: "name",
|
||||
Target: "environment",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
|
|
@ -7,11 +7,11 @@ import (
|
|||
func DefaultMutators() []bundle.Mutator {
|
||||
return []bundle.Mutator{
|
||||
ProcessRootIncludes(),
|
||||
DefineDefaultEnvironment(),
|
||||
DefineDefaultTarget(),
|
||||
LoadGitDetails(),
|
||||
}
|
||||
}
|
||||
|
||||
func DefaultMutatorsForEnvironment(env string) []bundle.Mutator {
|
||||
return append(DefaultMutators(), SelectEnvironment(env))
|
||||
func DefaultMutatorsForTarget(env string) []bundle.Mutator {
|
||||
return append(DefaultMutators(), SelectTarget(env))
|
||||
}
|
||||
|
|
|
@ -35,7 +35,7 @@ func overrideJobCompute(j *resources.Job, compute string) {
|
|||
func (m *overrideCompute) Apply(ctx context.Context, b *bundle.Bundle) error {
|
||||
if b.Config.Bundle.Mode != config.Development {
|
||||
if b.Config.Bundle.ComputeID != "" {
|
||||
return fmt.Errorf("cannot override compute for an environment that does not use 'mode: development'")
|
||||
return fmt.Errorf("cannot override compute for an target that does not use 'mode: development'")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -13,16 +13,16 @@ import (
|
|||
"github.com/databricks/databricks-sdk-go/service/ml"
|
||||
)
|
||||
|
||||
type processEnvironmentMode struct{}
|
||||
type processTargetMode struct{}
|
||||
|
||||
const developmentConcurrentRuns = 4
|
||||
|
||||
func ProcessEnvironmentMode() bundle.Mutator {
|
||||
return &processEnvironmentMode{}
|
||||
func ProcessTargetMode() bundle.Mutator {
|
||||
return &processTargetMode{}
|
||||
}
|
||||
|
||||
func (m *processEnvironmentMode) Name() string {
|
||||
return "ProcessEnvironmentMode"
|
||||
func (m *processTargetMode) Name() string {
|
||||
return "ProcessTargetMode"
|
||||
}
|
||||
|
||||
// Mark all resources as being for 'development' purposes, i.e.
|
||||
|
@ -110,14 +110,14 @@ func findIncorrectPath(b *bundle.Bundle, mode config.Mode) string {
|
|||
|
||||
func validateProductionMode(ctx context.Context, b *bundle.Bundle, isPrincipalUsed bool) error {
|
||||
if b.Config.Bundle.Git.Inferred {
|
||||
env := b.Config.Bundle.Environment
|
||||
return fmt.Errorf("environment with 'mode: production' must specify an explicit 'environments.%s.git' configuration", env)
|
||||
env := b.Config.Bundle.Target
|
||||
return fmt.Errorf("target with 'mode: production' must specify an explicit 'targets.%s.git' configuration", env)
|
||||
}
|
||||
|
||||
r := b.Config.Resources
|
||||
for i := range r.Pipelines {
|
||||
if r.Pipelines[i].Development {
|
||||
return fmt.Errorf("environment with 'mode: production' cannot specify a pipeline with 'development: true'")
|
||||
return fmt.Errorf("target with 'mode: production' cannot specify a pipeline with 'development: true'")
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -125,7 +125,7 @@ func validateProductionMode(ctx context.Context, b *bundle.Bundle, isPrincipalUs
|
|||
if path := findIncorrectPath(b, config.Production); path != "" {
|
||||
message := "%s must not contain the current username when using 'mode: production'"
|
||||
if path == "root_path" {
|
||||
return fmt.Errorf(message+"\n tip: set workspace.root_path to a shared path such as /Shared/.bundle/${bundle.name}/${bundle.environment}", path)
|
||||
return fmt.Errorf(message+"\n tip: set workspace.root_path to a shared path such as /Shared/.bundle/${bundle.name}/${bundle.target}", path)
|
||||
} else {
|
||||
return fmt.Errorf(message, path)
|
||||
}
|
||||
|
@ -165,7 +165,7 @@ func isRunAsSet(r config.Resources) bool {
|
|||
return true
|
||||
}
|
||||
|
||||
func (m *processEnvironmentMode) Apply(ctx context.Context, b *bundle.Bundle) error {
|
||||
func (m *processTargetMode) Apply(ctx context.Context, b *bundle.Bundle) error {
|
||||
switch b.Config.Bundle.Mode {
|
||||
case config.Development:
|
||||
err := validateDevelopmentMode(b)
|
|
@ -58,10 +58,10 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
|
|||
}
|
||||
}
|
||||
|
||||
func TestProcessEnvironmentModeDevelopment(t *testing.T) {
|
||||
func TestProcessTargetModeDevelopment(t *testing.T) {
|
||||
bundle := mockBundle(config.Development)
|
||||
|
||||
m := ProcessEnvironmentMode()
|
||||
m := ProcessTargetMode()
|
||||
err := m.Apply(context.Background(), bundle)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "[dev lennart] job1", bundle.Config.Resources.Jobs["job1"].Name)
|
||||
|
@ -73,10 +73,10 @@ func TestProcessEnvironmentModeDevelopment(t *testing.T) {
|
|||
assert.True(t, bundle.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
|
||||
}
|
||||
|
||||
func TestProcessEnvironmentModeDefault(t *testing.T) {
|
||||
func TestProcessTargetModeDefault(t *testing.T) {
|
||||
bundle := mockBundle("")
|
||||
|
||||
m := ProcessEnvironmentMode()
|
||||
m := ProcessTargetMode()
|
||||
err := m.Apply(context.Background(), bundle)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "job1", bundle.Config.Resources.Jobs["job1"].Name)
|
||||
|
@ -84,7 +84,7 @@ func TestProcessEnvironmentModeDefault(t *testing.T) {
|
|||
assert.False(t, bundle.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
|
||||
}
|
||||
|
||||
func TestProcessEnvironmentModeProduction(t *testing.T) {
|
||||
func TestProcessTargetModeProduction(t *testing.T) {
|
||||
bundle := mockBundle(config.Production)
|
||||
|
||||
err := validateProductionMode(context.Background(), bundle, false)
|
||||
|
@ -118,7 +118,7 @@ func TestProcessEnvironmentModeProduction(t *testing.T) {
|
|||
assert.False(t, bundle.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
|
||||
}
|
||||
|
||||
func TestProcessEnvironmentModeProductionGit(t *testing.T) {
|
||||
func TestProcessTargetModeProductionGit(t *testing.T) {
|
||||
bundle := mockBundle(config.Production)
|
||||
|
||||
// Pretend the user didn't set Git configuration explicitly
|
||||
|
@ -129,10 +129,10 @@ func TestProcessEnvironmentModeProductionGit(t *testing.T) {
|
|||
bundle.Config.Bundle.Git.Inferred = false
|
||||
}
|
||||
|
||||
func TestProcessEnvironmentModeProductionOkForPrincipal(t *testing.T) {
|
||||
func TestProcessTargetModeProductionOkForPrincipal(t *testing.T) {
|
||||
bundle := mockBundle(config.Production)
|
||||
|
||||
// Our environment has all kinds of problems when not using service principals ...
|
||||
// Our target has all kinds of problems when not using service principals ...
|
||||
err := validateProductionMode(context.Background(), bundle, false)
|
||||
require.Error(t, err)
|
||||
|
||||
|
@ -152,7 +152,7 @@ func TestAllResourcesMocked(t *testing.T) {
|
|||
assert.True(
|
||||
t,
|
||||
!field.IsNil() && field.Len() > 0,
|
||||
"process_environment_mode should support '%s' (please add it to process_environment_mode.go and extend the test suite)",
|
||||
"process_target_mode should support '%s' (please add it to process_target_mode.go and extend the test suite)",
|
||||
resources.Type().Field(i).Name,
|
||||
)
|
||||
}
|
||||
|
@ -164,7 +164,7 @@ func TestAllResourcesRenamed(t *testing.T) {
|
|||
bundle := mockBundle(config.Development)
|
||||
resources := reflect.ValueOf(bundle.Config.Resources)
|
||||
|
||||
m := ProcessEnvironmentMode()
|
||||
m := ProcessTargetMode()
|
||||
err := m.Apply(context.Background(), bundle)
|
||||
require.NoError(t, err)
|
||||
|
||||
|
@ -179,7 +179,7 @@ func TestAllResourcesRenamed(t *testing.T) {
|
|||
assert.True(
|
||||
t,
|
||||
strings.Contains(nameField.String(), "dev"),
|
||||
"process_environment_mode should rename '%s' in '%s'",
|
||||
"process_target_mode should rename '%s' in '%s'",
|
||||
key,
|
||||
resources.Type().Field(i).Name,
|
||||
)
|
|
@ -1,54 +0,0 @@
|
|||
package mutator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"golang.org/x/exp/maps"
|
||||
)
|
||||
|
||||
type selectDefaultEnvironment struct{}
|
||||
|
||||
// SelectDefaultEnvironment merges the default environment into the root configuration.
|
||||
func SelectDefaultEnvironment() bundle.Mutator {
|
||||
return &selectDefaultEnvironment{}
|
||||
}
|
||||
|
||||
func (m *selectDefaultEnvironment) Name() string {
|
||||
return "SelectDefaultEnvironment"
|
||||
}
|
||||
|
||||
func (m *selectDefaultEnvironment) Apply(ctx context.Context, b *bundle.Bundle) error {
|
||||
if len(b.Config.Environments) == 0 {
|
||||
return fmt.Errorf("no environments defined")
|
||||
}
|
||||
|
||||
// One environment means there's only one default.
|
||||
names := maps.Keys(b.Config.Environments)
|
||||
if len(names) == 1 {
|
||||
return SelectEnvironment(names[0]).Apply(ctx, b)
|
||||
}
|
||||
|
||||
// Multiple environments means we look for the `default` flag.
|
||||
var defaults []string
|
||||
for name, env := range b.Config.Environments {
|
||||
if env != nil && env.Default {
|
||||
defaults = append(defaults, name)
|
||||
}
|
||||
}
|
||||
|
||||
// It is invalid to have multiple environments with the `default` flag set.
|
||||
if len(defaults) > 1 {
|
||||
return fmt.Errorf("multiple environments are marked as default (%s)", strings.Join(defaults, ", "))
|
||||
}
|
||||
|
||||
// If no environment has the `default` flag set, ask the user to specify one.
|
||||
if len(defaults) == 0 {
|
||||
return fmt.Errorf("please specify environment")
|
||||
}
|
||||
|
||||
// One default remaining.
|
||||
return SelectEnvironment(defaults[0]).Apply(ctx, b)
|
||||
}
|
|
@ -1,90 +0,0 @@
|
|||
package mutator_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"github.com/databricks/cli/bundle/config"
|
||||
"github.com/databricks/cli/bundle/config/mutator"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestSelectDefaultEnvironmentNoEnvironments(t *testing.T) {
|
||||
bundle := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Environments: map[string]*config.Environment{},
|
||||
},
|
||||
}
|
||||
err := mutator.SelectDefaultEnvironment().Apply(context.Background(), bundle)
|
||||
assert.ErrorContains(t, err, "no environments defined")
|
||||
}
|
||||
|
||||
func TestSelectDefaultEnvironmentSingleEnvironments(t *testing.T) {
|
||||
bundle := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Environments: map[string]*config.Environment{
|
||||
"foo": {},
|
||||
},
|
||||
},
|
||||
}
|
||||
err := mutator.SelectDefaultEnvironment().Apply(context.Background(), bundle)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "foo", bundle.Config.Bundle.Environment)
|
||||
}
|
||||
|
||||
func TestSelectDefaultEnvironmentNoDefaults(t *testing.T) {
|
||||
bundle := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Environments: map[string]*config.Environment{
|
||||
"foo": {},
|
||||
"bar": {},
|
||||
"qux": {},
|
||||
},
|
||||
},
|
||||
}
|
||||
err := mutator.SelectDefaultEnvironment().Apply(context.Background(), bundle)
|
||||
assert.ErrorContains(t, err, "please specify environment")
|
||||
}
|
||||
|
||||
func TestSelectDefaultEnvironmentNoDefaultsWithNil(t *testing.T) {
|
||||
bundle := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Environments: map[string]*config.Environment{
|
||||
"foo": nil,
|
||||
"bar": nil,
|
||||
},
|
||||
},
|
||||
}
|
||||
err := mutator.SelectDefaultEnvironment().Apply(context.Background(), bundle)
|
||||
assert.ErrorContains(t, err, "please specify environment")
|
||||
}
|
||||
|
||||
func TestSelectDefaultEnvironmentMultipleDefaults(t *testing.T) {
|
||||
bundle := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Environments: map[string]*config.Environment{
|
||||
"foo": {Default: true},
|
||||
"bar": {Default: true},
|
||||
"qux": {Default: true},
|
||||
},
|
||||
},
|
||||
}
|
||||
err := mutator.SelectDefaultEnvironment().Apply(context.Background(), bundle)
|
||||
assert.ErrorContains(t, err, "multiple environments are marked as default")
|
||||
}
|
||||
|
||||
func TestSelectDefaultEnvironmentSingleDefault(t *testing.T) {
|
||||
bundle := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Environments: map[string]*config.Environment{
|
||||
"foo": {},
|
||||
"bar": {Default: true},
|
||||
"qux": {},
|
||||
},
|
||||
},
|
||||
}
|
||||
err := mutator.SelectDefaultEnvironment().Apply(context.Background(), bundle)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "bar", bundle.Config.Bundle.Environment)
|
||||
}
|
|
@ -0,0 +1,54 @@
|
|||
package mutator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"golang.org/x/exp/maps"
|
||||
)
|
||||
|
||||
type selectDefaultTarget struct{}
|
||||
|
||||
// SelectDefaultTarget merges the default target into the root configuration.
|
||||
func SelectDefaultTarget() bundle.Mutator {
|
||||
return &selectDefaultTarget{}
|
||||
}
|
||||
|
||||
func (m *selectDefaultTarget) Name() string {
|
||||
return "SelectDefaultTarget"
|
||||
}
|
||||
|
||||
func (m *selectDefaultTarget) Apply(ctx context.Context, b *bundle.Bundle) error {
|
||||
if len(b.Config.Targets) == 0 {
|
||||
return fmt.Errorf("no targets defined")
|
||||
}
|
||||
|
||||
// One target means there's only one default.
|
||||
names := maps.Keys(b.Config.Targets)
|
||||
if len(names) == 1 {
|
||||
return SelectTarget(names[0]).Apply(ctx, b)
|
||||
}
|
||||
|
||||
// Multiple targets means we look for the `default` flag.
|
||||
var defaults []string
|
||||
for name, env := range b.Config.Targets {
|
||||
if env != nil && env.Default {
|
||||
defaults = append(defaults, name)
|
||||
}
|
||||
}
|
||||
|
||||
// It is invalid to have multiple targets with the `default` flag set.
|
||||
if len(defaults) > 1 {
|
||||
return fmt.Errorf("multiple targets are marked as default (%s)", strings.Join(defaults, ", "))
|
||||
}
|
||||
|
||||
// If no target has the `default` flag set, ask the user to specify one.
|
||||
if len(defaults) == 0 {
|
||||
return fmt.Errorf("please specify target")
|
||||
}
|
||||
|
||||
// One default remaining.
|
||||
return SelectTarget(defaults[0]).Apply(ctx, b)
|
||||
}
|
|
@ -0,0 +1,90 @@
|
|||
package mutator_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"github.com/databricks/cli/bundle/config"
|
||||
"github.com/databricks/cli/bundle/config/mutator"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestSelectDefaultTargetNoTargets(t *testing.T) {
|
||||
bundle := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Targets: map[string]*config.Target{},
|
||||
},
|
||||
}
|
||||
err := mutator.SelectDefaultTarget().Apply(context.Background(), bundle)
|
||||
assert.ErrorContains(t, err, "no targets defined")
|
||||
}
|
||||
|
||||
func TestSelectDefaultTargetSingleTargets(t *testing.T) {
|
||||
bundle := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Targets: map[string]*config.Target{
|
||||
"foo": {},
|
||||
},
|
||||
},
|
||||
}
|
||||
err := mutator.SelectDefaultTarget().Apply(context.Background(), bundle)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "foo", bundle.Config.Bundle.Target)
|
||||
}
|
||||
|
||||
func TestSelectDefaultTargetNoDefaults(t *testing.T) {
|
||||
bundle := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Targets: map[string]*config.Target{
|
||||
"foo": {},
|
||||
"bar": {},
|
||||
"qux": {},
|
||||
},
|
||||
},
|
||||
}
|
||||
err := mutator.SelectDefaultTarget().Apply(context.Background(), bundle)
|
||||
assert.ErrorContains(t, err, "please specify target")
|
||||
}
|
||||
|
||||
func TestSelectDefaultTargetNoDefaultsWithNil(t *testing.T) {
|
||||
bundle := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Targets: map[string]*config.Target{
|
||||
"foo": nil,
|
||||
"bar": nil,
|
||||
},
|
||||
},
|
||||
}
|
||||
err := mutator.SelectDefaultTarget().Apply(context.Background(), bundle)
|
||||
assert.ErrorContains(t, err, "please specify target")
|
||||
}
|
||||
|
||||
func TestSelectDefaultTargetMultipleDefaults(t *testing.T) {
|
||||
bundle := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Targets: map[string]*config.Target{
|
||||
"foo": {Default: true},
|
||||
"bar": {Default: true},
|
||||
"qux": {Default: true},
|
||||
},
|
||||
},
|
||||
}
|
||||
err := mutator.SelectDefaultTarget().Apply(context.Background(), bundle)
|
||||
assert.ErrorContains(t, err, "multiple targets are marked as default")
|
||||
}
|
||||
|
||||
func TestSelectDefaultTargetSingleDefault(t *testing.T) {
|
||||
bundle := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Targets: map[string]*config.Target{
|
||||
"foo": {},
|
||||
"bar": {Default: true},
|
||||
"qux": {},
|
||||
},
|
||||
},
|
||||
}
|
||||
err := mutator.SelectDefaultTarget().Apply(context.Background(), bundle)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "bar", bundle.Config.Bundle.Target)
|
||||
}
|
|
@ -1,48 +0,0 @@
|
|||
package mutator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
)
|
||||
|
||||
type selectEnvironment struct {
|
||||
name string
|
||||
}
|
||||
|
||||
// SelectEnvironment merges the specified environment into the root configuration.
|
||||
func SelectEnvironment(name string) bundle.Mutator {
|
||||
return &selectEnvironment{
|
||||
name: name,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *selectEnvironment) Name() string {
|
||||
return fmt.Sprintf("SelectEnvironment(%s)", m.name)
|
||||
}
|
||||
|
||||
func (m *selectEnvironment) Apply(_ context.Context, b *bundle.Bundle) error {
|
||||
if b.Config.Environments == nil {
|
||||
return fmt.Errorf("no environments defined")
|
||||
}
|
||||
|
||||
// Get specified environment
|
||||
env, ok := b.Config.Environments[m.name]
|
||||
if !ok {
|
||||
return fmt.Errorf("%s: no such environment", m.name)
|
||||
}
|
||||
|
||||
// Merge specified environment into root configuration structure.
|
||||
err := b.Config.MergeEnvironment(env)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Store specified environment in configuration for reference.
|
||||
b.Config.Bundle.Environment = m.name
|
||||
|
||||
// Clear environments after loading.
|
||||
b.Config.Environments = nil
|
||||
return nil
|
||||
}
|
|
@ -0,0 +1,54 @@
|
|||
package mutator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
)
|
||||
|
||||
type selectTarget struct {
|
||||
name string
|
||||
}
|
||||
|
||||
// SelectTarget merges the specified target into the root configuration.
|
||||
func SelectTarget(name string) bundle.Mutator {
|
||||
return &selectTarget{
|
||||
name: name,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *selectTarget) Name() string {
|
||||
return fmt.Sprintf("SelectTarget(%s)", m.name)
|
||||
}
|
||||
|
||||
func (m *selectTarget) Apply(_ context.Context, b *bundle.Bundle) error {
|
||||
if b.Config.Targets == nil {
|
||||
return fmt.Errorf("no targets defined")
|
||||
}
|
||||
|
||||
// Get specified target
|
||||
target, ok := b.Config.Targets[m.name]
|
||||
if !ok {
|
||||
return fmt.Errorf("%s: no such target", m.name)
|
||||
}
|
||||
|
||||
// Merge specified target into root configuration structure.
|
||||
err := b.Config.MergeTargetOverrides(target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Store specified target in configuration for reference.
|
||||
b.Config.Bundle.Target = m.name
|
||||
|
||||
// We do this for backward compatibility.
|
||||
// TODO: remove when Environments section is not supported anymore.
|
||||
b.Config.Bundle.Environment = b.Config.Bundle.Target
|
||||
|
||||
// Clear targets after loading.
|
||||
b.Config.Targets = nil
|
||||
b.Config.Environments = nil
|
||||
|
||||
return nil
|
||||
}
|
|
@ -11,13 +11,13 @@ import (
|
|||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSelectEnvironment(t *testing.T) {
|
||||
func TestSelectTarget(t *testing.T) {
|
||||
bundle := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Workspace: config.Workspace{
|
||||
Host: "foo",
|
||||
},
|
||||
Environments: map[string]*config.Environment{
|
||||
Targets: map[string]*config.Target{
|
||||
"default": {
|
||||
Workspace: &config.Workspace{
|
||||
Host: "bar",
|
||||
|
@ -26,19 +26,19 @@ func TestSelectEnvironment(t *testing.T) {
|
|||
},
|
||||
},
|
||||
}
|
||||
err := mutator.SelectEnvironment("default").Apply(context.Background(), bundle)
|
||||
err := mutator.SelectTarget("default").Apply(context.Background(), bundle)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "bar", bundle.Config.Workspace.Host)
|
||||
}
|
||||
|
||||
func TestSelectEnvironmentNotFound(t *testing.T) {
|
||||
func TestSelectTargetNotFound(t *testing.T) {
|
||||
bundle := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Environments: map[string]*config.Environment{
|
||||
Targets: map[string]*config.Target{
|
||||
"default": {},
|
||||
},
|
||||
},
|
||||
}
|
||||
err := mutator.SelectEnvironment("doesnt-exist").Apply(context.Background(), bundle)
|
||||
require.Error(t, err, "no environments defined")
|
||||
err := mutator.SelectTarget("doesnt-exist").Apply(context.Background(), bundle)
|
||||
require.Error(t, err, "no targets defined")
|
||||
}
|
|
@ -115,7 +115,7 @@ func (r *Resources) SetConfigFilePath(path string) {
|
|||
}
|
||||
|
||||
// MergeJobClusters iterates over all jobs and merges their job clusters.
|
||||
// This is called after applying the environment overrides.
|
||||
// This is called after applying the target overrides.
|
||||
func (r *Resources) MergeJobClusters() error {
|
||||
for _, job := range r.Jobs {
|
||||
if err := job.MergeJobClusters(); err != nil {
|
||||
|
|
|
@ -22,7 +22,7 @@ func (j *Job) MergeJobClusters() error {
|
|||
keys := make(map[string]*jobs.JobCluster)
|
||||
output := make([]jobs.JobCluster, 0, len(j.JobClusters))
|
||||
|
||||
// Environment overrides are always appended, so we can iterate in natural order to
|
||||
// Target overrides are always appended, so we can iterate in natural order to
|
||||
// first find the base definition, and merge instances we encounter later.
|
||||
for i := range j.JobClusters {
|
||||
key := j.JobClusters[i].JobClusterKey
|
||||
|
|
|
@ -69,11 +69,14 @@ type Root struct {
|
|||
// to deploy in this bundle (e.g. jobs, pipelines, etc.).
|
||||
Resources Resources `json:"resources,omitempty"`
|
||||
|
||||
// Environments can be used to differentiate settings and resources between
|
||||
// bundle deployment environments (e.g. development, staging, production).
|
||||
// Targets can be used to differentiate settings and resources between
|
||||
// bundle deployment targets (e.g. development, staging, production).
|
||||
// If not specified, the code below initializes this field with a
|
||||
// single default-initialized environment called "default".
|
||||
Environments map[string]*Environment `json:"environments,omitempty"`
|
||||
// single default-initialized target called "default".
|
||||
Targets map[string]*Target `json:"targets,omitempty"`
|
||||
|
||||
// DEPRECATED. Left for backward compatibility with Targets
|
||||
Environments map[string]*Target `json:"environments,omitempty"`
|
||||
}
|
||||
|
||||
func Load(path string) (*Root, error) {
|
||||
|
@ -103,8 +106,8 @@ func Load(path string) (*Root, error) {
|
|||
// was loaded from in configuration leafs that require it.
|
||||
func (r *Root) SetConfigFilePath(path string) {
|
||||
r.Resources.SetConfigFilePath(path)
|
||||
if r.Environments != nil {
|
||||
for _, env := range r.Environments {
|
||||
if r.Targets != nil {
|
||||
for _, env := range r.Targets {
|
||||
if env == nil {
|
||||
continue
|
||||
}
|
||||
|
@ -148,6 +151,15 @@ func (r *Root) Load(path string) error {
|
|||
return fmt.Errorf("failed to load %s: %w", path, err)
|
||||
}
|
||||
|
||||
if r.Environments != nil && r.Targets != nil {
|
||||
return fmt.Errorf("both 'environments' and 'targets' are specified, only 'targets' should be used: %s", path)
|
||||
}
|
||||
|
||||
if r.Environments != nil {
|
||||
//TODO: add a command line notice that this is a deprecated option.
|
||||
r.Targets = r.Environments
|
||||
}
|
||||
|
||||
r.Path = filepath.Dir(path)
|
||||
r.SetConfigFilePath(path)
|
||||
|
||||
|
@ -169,37 +181,37 @@ func (r *Root) Merge(other *Root) error {
|
|||
return mergo.Merge(r, other, mergo.WithOverride)
|
||||
}
|
||||
|
||||
func (r *Root) MergeEnvironment(env *Environment) error {
|
||||
func (r *Root) MergeTargetOverrides(target *Target) error {
|
||||
var err error
|
||||
|
||||
// Environment may be nil if it's empty.
|
||||
if env == nil {
|
||||
// Target may be nil if it's empty.
|
||||
if target == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if env.Bundle != nil {
|
||||
err = mergo.Merge(&r.Bundle, env.Bundle, mergo.WithOverride)
|
||||
if target.Bundle != nil {
|
||||
err = mergo.Merge(&r.Bundle, target.Bundle, mergo.WithOverride)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if env.Workspace != nil {
|
||||
err = mergo.Merge(&r.Workspace, env.Workspace, mergo.WithOverride)
|
||||
if target.Workspace != nil {
|
||||
err = mergo.Merge(&r.Workspace, target.Workspace, mergo.WithOverride)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if env.Artifacts != nil {
|
||||
err = mergo.Merge(&r.Artifacts, env.Artifacts, mergo.WithOverride, mergo.WithAppendSlice)
|
||||
if target.Artifacts != nil {
|
||||
err = mergo.Merge(&r.Artifacts, target.Artifacts, mergo.WithOverride, mergo.WithAppendSlice)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if env.Resources != nil {
|
||||
err = mergo.Merge(&r.Resources, env.Resources, mergo.WithOverride, mergo.WithAppendSlice)
|
||||
if target.Resources != nil {
|
||||
err = mergo.Merge(&r.Resources, target.Resources, mergo.WithOverride, mergo.WithAppendSlice)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -210,8 +222,8 @@ func (r *Root) MergeEnvironment(env *Environment) error {
|
|||
}
|
||||
}
|
||||
|
||||
if env.Variables != nil {
|
||||
for k, v := range env.Variables {
|
||||
if target.Variables != nil {
|
||||
for k, v := range target.Variables {
|
||||
variable, ok := r.Variables[k]
|
||||
if !ok {
|
||||
return fmt.Errorf("variable %s is not defined but is assigned a value", k)
|
||||
|
@ -222,24 +234,24 @@ func (r *Root) MergeEnvironment(env *Environment) error {
|
|||
}
|
||||
}
|
||||
|
||||
if env.Mode != "" {
|
||||
r.Bundle.Mode = env.Mode
|
||||
if target.Mode != "" {
|
||||
r.Bundle.Mode = target.Mode
|
||||
}
|
||||
|
||||
if env.ComputeID != "" {
|
||||
r.Bundle.ComputeID = env.ComputeID
|
||||
if target.ComputeID != "" {
|
||||
r.Bundle.ComputeID = target.ComputeID
|
||||
}
|
||||
|
||||
git := &r.Bundle.Git
|
||||
if env.Git.Branch != "" {
|
||||
git.Branch = env.Git.Branch
|
||||
if target.Git.Branch != "" {
|
||||
git.Branch = target.Git.Branch
|
||||
git.Inferred = false
|
||||
}
|
||||
if env.Git.Commit != "" {
|
||||
git.Commit = env.Git.Commit
|
||||
if target.Git.Commit != "" {
|
||||
git.Commit = target.Git.Commit
|
||||
}
|
||||
if env.Git.OriginURL != "" {
|
||||
git.OriginURL = env.Git.OriginURL
|
||||
if target.Git.OriginURL != "" {
|
||||
git.OriginURL = target.Git.OriginURL
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
|
@ -57,7 +57,7 @@ func TestRootMergeStruct(t *testing.T) {
|
|||
func TestRootMergeMap(t *testing.T) {
|
||||
root := &Root{
|
||||
Path: "path",
|
||||
Environments: map[string]*Environment{
|
||||
Targets: map[string]*Target{
|
||||
"development": {
|
||||
Workspace: &Workspace{
|
||||
Host: "foo",
|
||||
|
@ -68,7 +68,7 @@ func TestRootMergeMap(t *testing.T) {
|
|||
}
|
||||
other := &Root{
|
||||
Path: "path",
|
||||
Environments: map[string]*Environment{
|
||||
Targets: map[string]*Target{
|
||||
"development": {
|
||||
Workspace: &Workspace{
|
||||
Host: "bar",
|
||||
|
@ -77,7 +77,7 @@ func TestRootMergeMap(t *testing.T) {
|
|||
},
|
||||
}
|
||||
assert.NoError(t, root.Merge(other))
|
||||
assert.Equal(t, &Workspace{Host: "bar", Profile: "profile"}, root.Environments["development"].Workspace)
|
||||
assert.Equal(t, &Workspace{Host: "bar", Profile: "profile"}, root.Targets["development"].Workspace)
|
||||
}
|
||||
|
||||
func TestDuplicateIdOnLoadReturnsError(t *testing.T) {
|
||||
|
@ -159,12 +159,12 @@ func TestInitializeVariablesUndefinedVariables(t *testing.T) {
|
|||
assert.ErrorContains(t, err, "variable bar has not been defined")
|
||||
}
|
||||
|
||||
func TestRootMergeEnvironmentWithMode(t *testing.T) {
|
||||
func TestRootMergeTargetOverridesWithMode(t *testing.T) {
|
||||
root := &Root{
|
||||
Bundle: Bundle{},
|
||||
}
|
||||
env := &Environment{Mode: Development}
|
||||
require.NoError(t, root.MergeEnvironment(env))
|
||||
env := &Target{Mode: Development}
|
||||
require.NoError(t, root.MergeTargetOverrides(env))
|
||||
assert.Equal(t, Development, root.Bundle.Mode)
|
||||
}
|
||||
|
||||
|
|
|
@ -2,14 +2,14 @@ package config
|
|||
|
||||
type Mode string
|
||||
|
||||
// Environment defines overrides for a single environment.
|
||||
// Target defines overrides for a single target.
|
||||
// This structure is recursively merged into the root configuration.
|
||||
type Environment struct {
|
||||
// Default marks that this environment must be used if one isn't specified
|
||||
// by the user (through environment variable or command line argument).
|
||||
type Target struct {
|
||||
// Default marks that this target must be used if one isn't specified
|
||||
// by the user (through target variable or command line argument).
|
||||
Default bool `json:"default,omitempty"`
|
||||
|
||||
// Determines the mode of the environment.
|
||||
// Determines the mode of the target.
|
||||
// For example, 'mode: development' can be used for deployments for
|
||||
// development purposes.
|
||||
Mode Mode `json:"mode,omitempty"`
|
||||
|
@ -27,7 +27,7 @@ type Environment struct {
|
|||
|
||||
// Override default values for defined variables
|
||||
// Does not permit defining new variables or redefining existing ones
|
||||
// in the scope of an environment
|
||||
// in the scope of an target
|
||||
Variables map[string]string `json:"variables,omitempty"`
|
||||
|
||||
Git Git `json:"git,omitempty"`
|
|
@ -18,7 +18,7 @@ type Variable struct {
|
|||
// resolved in the following priority order (from highest to lowest)
|
||||
//
|
||||
// 1. Command line flag. For example: `--var="foo=bar"`
|
||||
// 2. Environment variable. eg: BUNDLE_VAR_foo=bar
|
||||
// 2. Target variable. eg: BUNDLE_VAR_foo=bar
|
||||
// 3. Default value as defined in the applicable environments block
|
||||
// 4. Default value defined in variable definition
|
||||
// 5. Throw error, since if no default value is defined, then the variable
|
||||
|
|
|
@ -45,7 +45,7 @@ type Workspace struct {
|
|||
CurrentUser *User `json:"current_user,omitempty" bundle:"readonly"`
|
||||
|
||||
// Remote workspace base path for deployment state, for artifacts, as synchronization target.
|
||||
// This defaults to "~/.bundle/${bundle.name}/${bundle.environment}" where "~" expands to
|
||||
// This defaults to "~/.bundle/${bundle.name}/${bundle.target}" where "~" expands to
|
||||
// the current user's home directory in the workspace (e.g. `/Users/jane@doe.com`).
|
||||
RootPath string `json:"root_path,omitempty"`
|
||||
|
||||
|
|
|
@ -31,7 +31,7 @@ func TestInitEnvironmentVariables(t *testing.T) {
|
|||
Config: config.Root{
|
||||
Path: t.TempDir(),
|
||||
Bundle: config.Bundle{
|
||||
Environment: "whatever",
|
||||
Target: "whatever",
|
||||
Terraform: &config.Terraform{
|
||||
ExecPath: "terraform",
|
||||
},
|
||||
|
@ -58,7 +58,7 @@ func TestSetTempDirEnvVarsForUnixWithTmpDirSet(t *testing.T) {
|
|||
Config: config.Root{
|
||||
Path: t.TempDir(),
|
||||
Bundle: config.Bundle{
|
||||
Environment: "whatever",
|
||||
Target: "whatever",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
@ -86,7 +86,7 @@ func TestSetTempDirEnvVarsForUnixWithTmpDirNotSet(t *testing.T) {
|
|||
Config: config.Root{
|
||||
Path: t.TempDir(),
|
||||
Bundle: config.Bundle{
|
||||
Environment: "whatever",
|
||||
Target: "whatever",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
@ -112,7 +112,7 @@ func TestSetTempDirEnvVarsForWindowWithAllTmpDirEnvVarsSet(t *testing.T) {
|
|||
Config: config.Root{
|
||||
Path: t.TempDir(),
|
||||
Bundle: config.Bundle{
|
||||
Environment: "whatever",
|
||||
Target: "whatever",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
@ -142,7 +142,7 @@ func TestSetTempDirEnvVarsForWindowWithUserProfileAndTempSet(t *testing.T) {
|
|||
Config: config.Root{
|
||||
Path: t.TempDir(),
|
||||
Bundle: config.Bundle{
|
||||
Environment: "whatever",
|
||||
Target: "whatever",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
@ -172,7 +172,7 @@ func TestSetTempDirEnvVarsForWindowWithUserProfileSet(t *testing.T) {
|
|||
Config: config.Root{
|
||||
Path: t.TempDir(),
|
||||
Bundle: config.Bundle{
|
||||
Environment: "whatever",
|
||||
Target: "whatever",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
@ -202,7 +202,7 @@ func TestSetTempDirEnvVarsForWindowsWithoutAnyTempDirEnvVarsSet(t *testing.T) {
|
|||
Config: config.Root{
|
||||
Path: t.TempDir(),
|
||||
Bundle: config.Bundle{
|
||||
Environment: "whatever",
|
||||
Target: "whatever",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
@ -230,7 +230,7 @@ func TestSetProxyEnvVars(t *testing.T) {
|
|||
Config: config.Root{
|
||||
Path: t.TempDir(),
|
||||
Bundle: config.Bundle{
|
||||
Environment: "whatever",
|
||||
Target: "whatever",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
|
|
@ -20,7 +20,7 @@ func TestLoadWithNoState(t *testing.T) {
|
|||
Config: config.Root{
|
||||
Path: t.TempDir(),
|
||||
Bundle: config.Bundle{
|
||||
Environment: "whatever",
|
||||
Target: "whatever",
|
||||
Terraform: &config.Terraform{
|
||||
ExecPath: "terraform",
|
||||
},
|
||||
|
|
|
@ -26,7 +26,7 @@ func Initialize() bundle.Mutator {
|
|||
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
|
||||
),
|
||||
mutator.OverrideCompute(),
|
||||
mutator.ProcessEnvironmentMode(),
|
||||
mutator.ProcessTargetMode(),
|
||||
mutator.TranslatePaths(),
|
||||
terraform.Initialize(),
|
||||
},
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
`docs/bundle_descriptions.json` contains both autogenerated as well as manually written
|
||||
descriptions for the json schema. Specifically
|
||||
1. `resources` : almost all descriptions are autogenerated from the OpenAPI spec
|
||||
2. `environments` : almost all descriptions are copied over from root level entities (eg: `bundle`, `artifacts`)
|
||||
2. `targets` : almost all descriptions are copied over from root level entities (eg: `bundle`, `artifacts`)
|
||||
3. `bundle` : manually editted
|
||||
4. `include` : manually editted
|
||||
5. `workspace` : manually editted
|
||||
|
@ -17,7 +17,7 @@ These descriptions are rendered in the inline documentation in an IDE
|
|||
`databricks bundle schema --only-docs > ~/databricks/bundle/schema/docs/bundle_descriptions.json`
|
||||
2. Manually edit bundle_descriptions.json to add your descriptions
|
||||
3. Build again to embed the new `bundle_descriptions.json` into the binary (`go build`)
|
||||
4. Again run `databricks bundle schema --only-docs > ~/databricks/bundle/schema/docs/bundle_descriptions.json` to copy over any applicable descriptions to `environments`
|
||||
4. Again run `databricks bundle schema --only-docs > ~/databricks/bundle/schema/docs/bundle_descriptions.json` to copy over any applicable descriptions to `targets`
|
||||
5. push to repo
|
||||
|
||||
|
||||
|
|
|
@ -52,20 +52,20 @@ func BundleDocs(openapiSpecPath string) (*Docs, error) {
|
|||
}
|
||||
docs.Properties["resources"] = schemaToDocs(resourceSchema)
|
||||
}
|
||||
docs.refreshEnvironmentsDocs()
|
||||
docs.refreshTargetsDocs()
|
||||
return docs, nil
|
||||
}
|
||||
|
||||
func (docs *Docs) refreshEnvironmentsDocs() error {
|
||||
environmentsDocs, ok := docs.Properties["environments"]
|
||||
if !ok || environmentsDocs.AdditionalProperties == nil ||
|
||||
environmentsDocs.AdditionalProperties.Properties == nil {
|
||||
return fmt.Errorf("invalid environments descriptions")
|
||||
func (docs *Docs) refreshTargetsDocs() error {
|
||||
targetsDocs, ok := docs.Properties["targets"]
|
||||
if !ok || targetsDocs.AdditionalProperties == nil ||
|
||||
targetsDocs.AdditionalProperties.Properties == nil {
|
||||
return fmt.Errorf("invalid targets descriptions")
|
||||
}
|
||||
environmentProperties := environmentsDocs.AdditionalProperties.Properties
|
||||
targetProperties := targetsDocs.AdditionalProperties.Properties
|
||||
propertiesToCopy := []string{"artifacts", "bundle", "resources", "workspace"}
|
||||
for _, p := range propertiesToCopy {
|
||||
environmentProperties[p] = docs.Properties[p]
|
||||
targetProperties[p] = docs.Properties[p]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -36,7 +36,7 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"environments": {
|
||||
"targets": {
|
||||
"description": "",
|
||||
"additionalproperties": {
|
||||
"description": "",
|
||||
|
@ -1827,7 +1827,7 @@
|
|||
"description": "Connection profile to use. By default profiles are specified in ~/.databrickscfg."
|
||||
},
|
||||
"root_path": {
|
||||
"description": "The base location for synchronizing files, artifacts and state. Defaults to `/Users/jane@doe.com/.bundle/${bundle.name}/${bundle.environment}`"
|
||||
"description": "The base location for synchronizing files, artifacts and state. Defaults to `/Users/jane@doe.com/.bundle/${bundle.name}/${bundle.target}`"
|
||||
},
|
||||
"state_path": {
|
||||
"description": "The remote path to synchronize bundle state to. This defaults to `${workspace.root}/state`"
|
||||
|
@ -3591,7 +3591,7 @@
|
|||
"description": "Connection profile to use. By default profiles are specified in ~/.databrickscfg."
|
||||
},
|
||||
"root_path": {
|
||||
"description": "The base location for synchronizing files, artifacts and state. Defaults to `/Users/jane@doe.com/.bundle/${bundle.name}/${bundle.environment}`"
|
||||
"description": "The base location for synchronizing files, artifacts and state. Defaults to `/Users/jane@doe.com/.bundle/${bundle.name}/${bundle.target}`"
|
||||
},
|
||||
"state_path": {
|
||||
"description": "The remote path to synchronize bundle state to. This defaults to `${workspace.root}/state`"
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
bundle:
|
||||
name: autoload git config test
|
||||
|
||||
environments:
|
||||
targets:
|
||||
development:
|
||||
default: true
|
||||
|
||||
|
|
|
@ -1,5 +0,0 @@
|
|||
bundle:
|
||||
name: environment_empty
|
||||
|
||||
environments:
|
||||
development:
|
|
@ -1,12 +0,0 @@
|
|||
package config_tests
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestEnvironmentEmpty(t *testing.T) {
|
||||
b := loadEnvironment(t, "./environment_empty", "development")
|
||||
assert.Equal(t, "development", b.Config.Bundle.Environment)
|
||||
}
|
|
@ -0,0 +1,20 @@
|
|||
package config_tests
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGitAutoLoadWithEnvironment(t *testing.T) {
|
||||
b := load(t, "./environments_autoload_git")
|
||||
assert.True(t, b.Config.Bundle.Git.Inferred)
|
||||
assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli")
|
||||
}
|
||||
|
||||
func TestGitManuallySetBranchWithEnvironment(t *testing.T) {
|
||||
b := loadTarget(t, "./environments_autoload_git", "production")
|
||||
assert.False(t, b.Config.Bundle.Git.Inferred)
|
||||
assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
|
||||
assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli")
|
||||
}
|
|
@ -7,17 +7,17 @@ import (
|
|||
)
|
||||
|
||||
func TestEnvironmentOverridesWorkspaceDev(t *testing.T) {
|
||||
b := loadEnvironment(t, "./environment_overrides/workspace", "development")
|
||||
b := loadTarget(t, "./environment_overrides/workspace", "development")
|
||||
assert.Equal(t, "https://development.acme.cloud.databricks.com/", b.Config.Workspace.Host)
|
||||
}
|
||||
|
||||
func TestEnvironmentOverridesWorkspaceStaging(t *testing.T) {
|
||||
b := loadEnvironment(t, "./environment_overrides/workspace", "staging")
|
||||
b := loadTarget(t, "./environment_overrides/workspace", "staging")
|
||||
assert.Equal(t, "https://staging.acme.cloud.databricks.com/", b.Config.Workspace.Host)
|
||||
}
|
||||
|
||||
func TestEnvironmentOverridesResourcesDev(t *testing.T) {
|
||||
b := loadEnvironment(t, "./environment_overrides/resources", "development")
|
||||
b := loadTarget(t, "./environment_overrides/resources", "development")
|
||||
assert.Equal(t, "base job", b.Config.Resources.Jobs["job1"].Name)
|
||||
|
||||
// Base values are preserved in the development environment.
|
||||
|
@ -26,7 +26,7 @@ func TestEnvironmentOverridesResourcesDev(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestEnvironmentOverridesResourcesStaging(t *testing.T) {
|
||||
b := loadEnvironment(t, "./environment_overrides/resources", "staging")
|
||||
b := loadTarget(t, "./environment_overrides/resources", "staging")
|
||||
assert.Equal(t, "staging job", b.Config.Resources.Jobs["job1"].Name)
|
||||
|
||||
// Overrides are only applied if they are not zero-valued.
|
||||
|
|
|
@ -0,0 +1,11 @@
|
|||
bundle:
|
||||
name: autoload git config test
|
||||
|
||||
environments:
|
||||
development:
|
||||
default: true
|
||||
|
||||
production:
|
||||
# production can only be deployed from the 'main' branch
|
||||
git:
|
||||
branch: main
|
|
@ -0,0 +1,44 @@
|
|||
resources:
|
||||
pipelines:
|
||||
nyc_taxi_pipeline:
|
||||
name: "nyc taxi loader"
|
||||
libraries:
|
||||
- notebook:
|
||||
path: ./dlt/nyc_taxi_loader
|
||||
|
||||
environments:
|
||||
development:
|
||||
mode: development
|
||||
resources:
|
||||
pipelines:
|
||||
nyc_taxi_pipeline:
|
||||
target: nyc_taxi_development
|
||||
development: true
|
||||
|
||||
staging:
|
||||
resources:
|
||||
pipelines:
|
||||
nyc_taxi_pipeline:
|
||||
target: nyc_taxi_staging
|
||||
development: false
|
||||
|
||||
production:
|
||||
mode: production
|
||||
resources:
|
||||
pipelines:
|
||||
nyc_taxi_pipeline:
|
||||
target: nyc_taxi_production
|
||||
development: false
|
||||
photon: true
|
||||
|
||||
jobs:
|
||||
pipeline_schedule:
|
||||
name: Daily refresh of production pipeline
|
||||
|
||||
schedule:
|
||||
quartz_cron_expression: 6 6 11 * * ?
|
||||
timezone_id: UTC
|
||||
|
||||
tasks:
|
||||
- pipeline_task:
|
||||
pipeline_id: "to be interpolated"
|
|
@ -0,0 +1,56 @@
|
|||
package config_tests
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/databricks/cli/bundle/config"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestJobAndPipelineDevelopmentWithEnvironment(t *testing.T) {
|
||||
b := loadTarget(t, "./environments_job_and_pipeline", "development")
|
||||
assert.Len(t, b.Config.Resources.Jobs, 0)
|
||||
assert.Len(t, b.Config.Resources.Pipelines, 1)
|
||||
|
||||
p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"]
|
||||
assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath))
|
||||
assert.Equal(t, b.Config.Bundle.Mode, config.Development)
|
||||
assert.True(t, p.Development)
|
||||
require.Len(t, p.Libraries, 1)
|
||||
assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path)
|
||||
assert.Equal(t, "nyc_taxi_development", p.Target)
|
||||
}
|
||||
|
||||
func TestJobAndPipelineStagingWithEnvironment(t *testing.T) {
|
||||
b := loadTarget(t, "./environments_job_and_pipeline", "staging")
|
||||
assert.Len(t, b.Config.Resources.Jobs, 0)
|
||||
assert.Len(t, b.Config.Resources.Pipelines, 1)
|
||||
|
||||
p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"]
|
||||
assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath))
|
||||
assert.False(t, p.Development)
|
||||
require.Len(t, p.Libraries, 1)
|
||||
assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path)
|
||||
assert.Equal(t, "nyc_taxi_staging", p.Target)
|
||||
}
|
||||
|
||||
func TestJobAndPipelineProductionWithEnvironment(t *testing.T) {
|
||||
b := loadTarget(t, "./environments_job_and_pipeline", "production")
|
||||
assert.Len(t, b.Config.Resources.Jobs, 1)
|
||||
assert.Len(t, b.Config.Resources.Pipelines, 1)
|
||||
|
||||
p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"]
|
||||
assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath))
|
||||
assert.False(t, p.Development)
|
||||
require.Len(t, p.Libraries, 1)
|
||||
assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path)
|
||||
assert.Equal(t, "nyc_taxi_production", p.Target)
|
||||
|
||||
j := b.Config.Resources.Jobs["pipeline_schedule"]
|
||||
assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(j.ConfigFilePath))
|
||||
assert.Equal(t, "Daily refresh of production pipeline", j.Name)
|
||||
require.Len(t, j.Tasks, 1)
|
||||
assert.NotEmpty(t, j.Tasks[0].PipelineTask.PipelineId)
|
||||
}
|
|
@ -0,0 +1,35 @@
|
|||
bundle:
|
||||
name: override_job_cluster
|
||||
|
||||
workspace:
|
||||
host: https://acme.cloud.databricks.com/
|
||||
|
||||
resources:
|
||||
jobs:
|
||||
foo:
|
||||
name: job
|
||||
job_clusters:
|
||||
- job_cluster_key: key
|
||||
new_cluster:
|
||||
spark_version: 13.3.x-scala2.12
|
||||
|
||||
environments:
|
||||
development:
|
||||
resources:
|
||||
jobs:
|
||||
foo:
|
||||
job_clusters:
|
||||
- job_cluster_key: key
|
||||
new_cluster:
|
||||
node_type_id: i3.xlarge
|
||||
num_workers: 1
|
||||
|
||||
staging:
|
||||
resources:
|
||||
jobs:
|
||||
foo:
|
||||
job_clusters:
|
||||
- job_cluster_key: key
|
||||
new_cluster:
|
||||
node_type_id: i3.2xlarge
|
||||
num_workers: 4
|
|
@ -0,0 +1,29 @@
|
|||
package config_tests
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestOverrideJobClusterDevWithEnvironment(t *testing.T) {
|
||||
b := loadTarget(t, "./environments_override_job_cluster", "development")
|
||||
assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name)
|
||||
assert.Len(t, b.Config.Resources.Jobs["foo"].JobClusters, 1)
|
||||
|
||||
c := b.Config.Resources.Jobs["foo"].JobClusters[0]
|
||||
assert.Equal(t, "13.3.x-scala2.12", c.NewCluster.SparkVersion)
|
||||
assert.Equal(t, "i3.xlarge", c.NewCluster.NodeTypeId)
|
||||
assert.Equal(t, 1, c.NewCluster.NumWorkers)
|
||||
}
|
||||
|
||||
func TestOverrideJobClusterStagingWithEnvironment(t *testing.T) {
|
||||
b := loadTarget(t, "./environments_override_job_cluster", "staging")
|
||||
assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name)
|
||||
assert.Len(t, b.Config.Resources.Jobs["foo"].JobClusters, 1)
|
||||
|
||||
c := b.Config.Resources.Jobs["foo"].JobClusters[0]
|
||||
assert.Equal(t, "13.3.x-scala2.12", c.NewCluster.SparkVersion)
|
||||
assert.Equal(t, "i3.2xlarge", c.NewCluster.NodeTypeId)
|
||||
assert.Equal(t, 4, c.NewCluster.NumWorkers)
|
||||
}
|
|
@ -17,7 +17,7 @@ func TestGitAutoLoad(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestGitManuallySetBranch(t *testing.T) {
|
||||
b := loadEnvironment(t, "./autoload_git", "production")
|
||||
b := loadTarget(t, "./autoload_git", "production")
|
||||
assert.False(t, b.Config.Bundle.Git.Inferred)
|
||||
assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
|
||||
assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli")
|
||||
|
|
|
@ -0,0 +1,14 @@
|
|||
bundle:
|
||||
name: foo ${workspace.profile}
|
||||
|
||||
workspace:
|
||||
profile: bar
|
||||
|
||||
targets:
|
||||
development:
|
||||
default: true
|
||||
|
||||
resources:
|
||||
jobs:
|
||||
my_job:
|
||||
name: "${bundle.name} | ${workspace.profile} | ${bundle.environment} | ${bundle.target}"
|
|
@ -20,3 +20,15 @@ func TestInterpolation(t *testing.T) {
|
|||
assert.Equal(t, "foo bar", b.Config.Bundle.Name)
|
||||
assert.Equal(t, "foo bar | bar", b.Config.Resources.Jobs["my_job"].Name)
|
||||
}
|
||||
|
||||
func TestInterpolationWithTarget(t *testing.T) {
|
||||
b := loadTarget(t, "./interpolation_target", "development")
|
||||
err := bundle.Apply(context.Background(), b, interpolation.Interpolate(
|
||||
interpolation.IncludeLookupsInPath("bundle"),
|
||||
interpolation.IncludeLookupsInPath("workspace"),
|
||||
))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "foo bar", b.Config.Bundle.Name)
|
||||
assert.Equal(t, "foo bar | bar | development | development", b.Config.Resources.Jobs["my_job"].Name)
|
||||
|
||||
}
|
||||
|
|
|
@ -6,7 +6,7 @@ resources:
|
|||
- notebook:
|
||||
path: ./dlt/nyc_taxi_loader
|
||||
|
||||
environments:
|
||||
targets:
|
||||
development:
|
||||
mode: development
|
||||
resources:
|
||||
|
|
|
@ -10,7 +10,7 @@ import (
|
|||
)
|
||||
|
||||
func TestJobAndPipelineDevelopment(t *testing.T) {
|
||||
b := loadEnvironment(t, "./job_and_pipeline", "development")
|
||||
b := loadTarget(t, "./job_and_pipeline", "development")
|
||||
assert.Len(t, b.Config.Resources.Jobs, 0)
|
||||
assert.Len(t, b.Config.Resources.Pipelines, 1)
|
||||
|
||||
|
@ -24,7 +24,7 @@ func TestJobAndPipelineDevelopment(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestJobAndPipelineStaging(t *testing.T) {
|
||||
b := loadEnvironment(t, "./job_and_pipeline", "staging")
|
||||
b := loadTarget(t, "./job_and_pipeline", "staging")
|
||||
assert.Len(t, b.Config.Resources.Jobs, 0)
|
||||
assert.Len(t, b.Config.Resources.Pipelines, 1)
|
||||
|
||||
|
@ -37,7 +37,7 @@ func TestJobAndPipelineStaging(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestJobAndPipelineProduction(t *testing.T) {
|
||||
b := loadEnvironment(t, "./job_and_pipeline", "production")
|
||||
b := loadTarget(t, "./job_and_pipeline", "production")
|
||||
assert.Len(t, b.Config.Resources.Jobs, 1)
|
||||
assert.Len(t, b.Config.Resources.Pipelines, 1)
|
||||
|
||||
|
|
|
@ -18,9 +18,9 @@ func load(t *testing.T, path string) *bundle.Bundle {
|
|||
return b
|
||||
}
|
||||
|
||||
func loadEnvironment(t *testing.T, path, env string) *bundle.Bundle {
|
||||
func loadTarget(t *testing.T, path, env string) *bundle.Bundle {
|
||||
b := load(t, path)
|
||||
err := bundle.Apply(context.Background(), b, mutator.SelectEnvironment(env))
|
||||
err := bundle.Apply(context.Background(), b, mutator.SelectTarget(env))
|
||||
require.NoError(t, err)
|
||||
return b
|
||||
}
|
||||
|
|
|
@ -13,7 +13,7 @@ resources:
|
|||
new_cluster:
|
||||
spark_version: 13.3.x-scala2.12
|
||||
|
||||
environments:
|
||||
targets:
|
||||
development:
|
||||
resources:
|
||||
jobs:
|
||||
|
|
|
@ -7,7 +7,7 @@ import (
|
|||
)
|
||||
|
||||
func TestOverrideJobClusterDev(t *testing.T) {
|
||||
b := loadEnvironment(t, "./override_job_cluster", "development")
|
||||
b := loadTarget(t, "./override_job_cluster", "development")
|
||||
assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name)
|
||||
assert.Len(t, b.Config.Resources.Jobs["foo"].JobClusters, 1)
|
||||
|
||||
|
@ -18,7 +18,7 @@ func TestOverrideJobClusterDev(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestOverrideJobClusterStaging(t *testing.T) {
|
||||
b := loadEnvironment(t, "./override_job_cluster", "staging")
|
||||
b := loadTarget(t, "./override_job_cluster", "staging")
|
||||
assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name)
|
||||
assert.Len(t, b.Config.Resources.Jobs["foo"].JobClusters, 1)
|
||||
|
||||
|
|
|
@ -0,0 +1,5 @@
|
|||
bundle:
|
||||
name: target_empty
|
||||
|
||||
targets:
|
||||
development:
|
|
@ -0,0 +1,12 @@
|
|||
package config_tests
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestTargetEmpty(t *testing.T) {
|
||||
b := loadTarget(t, "./target_empty", "development")
|
||||
assert.Equal(t, "development", b.Config.Bundle.Target)
|
||||
}
|
|
@ -0,0 +1,20 @@
|
|||
bundle:
|
||||
name: environment_overrides
|
||||
|
||||
workspace:
|
||||
host: https://acme.cloud.databricks.com/
|
||||
|
||||
resources:
|
||||
jobs:
|
||||
job1:
|
||||
name: "base job"
|
||||
|
||||
targets:
|
||||
development:
|
||||
default: true
|
||||
|
||||
staging:
|
||||
resources:
|
||||
jobs:
|
||||
job1:
|
||||
name: "staging job"
|
|
@ -0,0 +1,14 @@
|
|||
bundle:
|
||||
name: environment_overrides
|
||||
|
||||
workspace:
|
||||
host: https://acme.cloud.databricks.com/
|
||||
|
||||
targets:
|
||||
development:
|
||||
workspace:
|
||||
host: https://development.acme.cloud.databricks.com/
|
||||
|
||||
staging:
|
||||
workspace:
|
||||
host: https://staging.acme.cloud.databricks.com/
|
|
@ -0,0 +1,27 @@
|
|||
package config_tests
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestTargetOverridesWorkspaceDev(t *testing.T) {
|
||||
b := loadTarget(t, "./target_overrides/workspace", "development")
|
||||
assert.Equal(t, "https://development.acme.cloud.databricks.com/", b.Config.Workspace.Host)
|
||||
}
|
||||
|
||||
func TestTargetOverridesWorkspaceStaging(t *testing.T) {
|
||||
b := loadTarget(t, "./target_overrides/workspace", "staging")
|
||||
assert.Equal(t, "https://staging.acme.cloud.databricks.com/", b.Config.Workspace.Host)
|
||||
}
|
||||
|
||||
func TestTargetOverridesResourcesDev(t *testing.T) {
|
||||
b := loadTarget(t, "./target_overrides/resources", "development")
|
||||
assert.Equal(t, "base job", b.Config.Resources.Jobs["job1"].Name)
|
||||
}
|
||||
|
||||
func TestTargetOverridesResourcesStaging(t *testing.T) {
|
||||
b := loadTarget(t, "./target_overrides/resources", "staging")
|
||||
assert.Equal(t, "staging job", b.Config.Resources.Jobs["job1"].Name)
|
||||
}
|
|
@ -12,7 +12,7 @@ bundle:
|
|||
workspace:
|
||||
profile: ${var.a} ${var.b}
|
||||
|
||||
environments:
|
||||
targets:
|
||||
env-with-single-variable-override:
|
||||
variables:
|
||||
b: dev-b
|
||||
|
|
|
@ -34,10 +34,10 @@ func TestVariablesLoadingFailsWhenRequiredVariableIsNotSpecified(t *testing.T) {
|
|||
assert.ErrorContains(t, err, "no value assigned to required variable b. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_b environment variable")
|
||||
}
|
||||
|
||||
func TestVariablesEnvironmentsBlockOverride(t *testing.T) {
|
||||
func TestVariablesTargetsBlockOverride(t *testing.T) {
|
||||
b := load(t, "./variables/env_overrides")
|
||||
err := bundle.Apply(context.Background(), b, bundle.Seq(
|
||||
mutator.SelectEnvironment("env-with-single-variable-override"),
|
||||
mutator.SelectTarget("env-with-single-variable-override"),
|
||||
mutator.SetVariables(),
|
||||
interpolation.Interpolate(
|
||||
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
|
||||
|
@ -46,10 +46,10 @@ func TestVariablesEnvironmentsBlockOverride(t *testing.T) {
|
|||
assert.Equal(t, "default-a dev-b", b.Config.Workspace.Profile)
|
||||
}
|
||||
|
||||
func TestVariablesEnvironmentsBlockOverrideForMultipleVariables(t *testing.T) {
|
||||
func TestVariablesTargetsBlockOverrideForMultipleVariables(t *testing.T) {
|
||||
b := load(t, "./variables/env_overrides")
|
||||
err := bundle.Apply(context.Background(), b, bundle.Seq(
|
||||
mutator.SelectEnvironment("env-with-two-variable-overrides"),
|
||||
mutator.SelectTarget("env-with-two-variable-overrides"),
|
||||
mutator.SetVariables(),
|
||||
interpolation.Interpolate(
|
||||
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
|
||||
|
@ -58,11 +58,11 @@ func TestVariablesEnvironmentsBlockOverrideForMultipleVariables(t *testing.T) {
|
|||
assert.Equal(t, "prod-a prod-b", b.Config.Workspace.Profile)
|
||||
}
|
||||
|
||||
func TestVariablesEnvironmentsBlockOverrideWithProcessEnvVars(t *testing.T) {
|
||||
func TestVariablesTargetsBlockOverrideWithProcessEnvVars(t *testing.T) {
|
||||
t.Setenv("BUNDLE_VAR_b", "env-var-b")
|
||||
b := load(t, "./variables/env_overrides")
|
||||
err := bundle.Apply(context.Background(), b, bundle.Seq(
|
||||
mutator.SelectEnvironment("env-with-two-variable-overrides"),
|
||||
mutator.SelectTarget("env-with-two-variable-overrides"),
|
||||
mutator.SetVariables(),
|
||||
interpolation.Interpolate(
|
||||
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
|
||||
|
@ -71,10 +71,10 @@ func TestVariablesEnvironmentsBlockOverrideWithProcessEnvVars(t *testing.T) {
|
|||
assert.Equal(t, "prod-a env-var-b", b.Config.Workspace.Profile)
|
||||
}
|
||||
|
||||
func TestVariablesEnvironmentsBlockOverrideWithMissingVariables(t *testing.T) {
|
||||
func TestVariablesTargetsBlockOverrideWithMissingVariables(t *testing.T) {
|
||||
b := load(t, "./variables/env_overrides")
|
||||
err := bundle.Apply(context.Background(), b, bundle.Seq(
|
||||
mutator.SelectEnvironment("env-missing-a-required-variable-assignment"),
|
||||
mutator.SelectTarget("env-missing-a-required-variable-assignment"),
|
||||
mutator.SetVariables(),
|
||||
interpolation.Interpolate(
|
||||
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
|
||||
|
@ -82,10 +82,10 @@ func TestVariablesEnvironmentsBlockOverrideWithMissingVariables(t *testing.T) {
|
|||
assert.ErrorContains(t, err, "no value assigned to required variable b. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_b environment variable")
|
||||
}
|
||||
|
||||
func TestVariablesEnvironmentsBlockOverrideWithUndefinedVariables(t *testing.T) {
|
||||
func TestVariablesTargetsBlockOverrideWithUndefinedVariables(t *testing.T) {
|
||||
b := load(t, "./variables/env_overrides")
|
||||
err := bundle.Apply(context.Background(), b, bundle.Seq(
|
||||
mutator.SelectEnvironment("env-using-an-undefined-variable"),
|
||||
mutator.SelectTarget("env-using-an-undefined-variable"),
|
||||
mutator.SetVariables(),
|
||||
interpolation.Interpolate(
|
||||
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
|
||||
|
|
|
@ -7,7 +7,7 @@ import (
|
|||
)
|
||||
|
||||
func ConfigureBundleWithVariables(cmd *cobra.Command, args []string) error {
|
||||
// Load bundle config and apply environment
|
||||
// Load bundle config and apply target
|
||||
err := root.MustConfigureBundle(cmd, args)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
@ -131,7 +131,7 @@ func newConfigureCommand() *cobra.Command {
|
|||
|
||||
// Include token flag for compatibility with the legacy CLI.
|
||||
// It doesn't actually do anything because we always use PATs.
|
||||
cmd.Flags().BoolP("token", "t", true, "Configure using Databricks Personal Access Token")
|
||||
cmd.Flags().Bool("token", true, "Configure using Databricks Personal Access Token")
|
||||
cmd.Flags().MarkHidden("token")
|
||||
|
||||
cmd.RunE = func(cmd *cobra.Command, args []string) error {
|
||||
|
|
|
@ -11,11 +11,12 @@ import (
|
|||
)
|
||||
|
||||
const envName = "DATABRICKS_BUNDLE_ENV"
|
||||
const targetName = "DATABRICKS_BUNDLE_TARGET"
|
||||
|
||||
// getEnvironment returns the name of the environment to operate in.
|
||||
func getEnvironment(cmd *cobra.Command) (value string) {
|
||||
// getTarget returns the name of the target to operate in.
|
||||
func getTarget(cmd *cobra.Command) (value string) {
|
||||
// The command line flag takes precedence.
|
||||
flag := cmd.Flag("environment")
|
||||
flag := cmd.Flag("target")
|
||||
if flag != nil {
|
||||
value = flag.Value.String()
|
||||
if value != "" {
|
||||
|
@ -23,8 +24,23 @@ func getEnvironment(cmd *cobra.Command) (value string) {
|
|||
}
|
||||
}
|
||||
|
||||
oldFlag := cmd.Flag("environment")
|
||||
if oldFlag != nil {
|
||||
value = flag.Value.String()
|
||||
if value != "" {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// If it's not set, use the environment variable.
|
||||
return os.Getenv(envName)
|
||||
target := os.Getenv(targetName)
|
||||
// If target env is not set with a new variable, try to check for old variable name
|
||||
// TODO: remove when environments section is not supported anymore
|
||||
if target == "" {
|
||||
target = os.Getenv(envName)
|
||||
}
|
||||
|
||||
return target
|
||||
}
|
||||
|
||||
func getProfile(cmd *cobra.Command) (value string) {
|
||||
|
@ -80,11 +96,11 @@ func configureBundle(cmd *cobra.Command, args []string, load func(ctx context.Co
|
|||
}
|
||||
|
||||
var m bundle.Mutator
|
||||
env := getEnvironment(cmd)
|
||||
env := getTarget(cmd)
|
||||
if env == "" {
|
||||
m = mutator.SelectDefaultEnvironment()
|
||||
m = mutator.SelectDefaultTarget()
|
||||
} else {
|
||||
m = mutator.SelectEnvironment(env)
|
||||
m = mutator.SelectTarget(env)
|
||||
}
|
||||
|
||||
ctx := cmd.Context()
|
||||
|
@ -108,19 +124,27 @@ func TryConfigureBundle(cmd *cobra.Command, args []string) error {
|
|||
return configureBundle(cmd, args, bundle.TryLoad)
|
||||
}
|
||||
|
||||
// environmentCompletion executes to autocomplete the argument to the environment flag.
|
||||
func environmentCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
||||
// targetCompletion executes to autocomplete the argument to the target flag.
|
||||
func targetCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
||||
b, err := loadBundle(cmd, args, bundle.MustLoad)
|
||||
if err != nil {
|
||||
cobra.CompErrorln(err.Error())
|
||||
return nil, cobra.ShellCompDirectiveError
|
||||
}
|
||||
|
||||
return maps.Keys(b.Config.Environments), cobra.ShellCompDirectiveDefault
|
||||
return maps.Keys(b.Config.Targets), cobra.ShellCompDirectiveDefault
|
||||
}
|
||||
|
||||
func initTargetFlag(cmd *cobra.Command) {
|
||||
// To operate in the context of a bundle, all commands must take an "target" parameter.
|
||||
cmd.PersistentFlags().StringP("target", "t", "", "bundle target to use (if applicable)")
|
||||
cmd.RegisterFlagCompletionFunc("target", targetCompletion)
|
||||
}
|
||||
|
||||
// DEPRECATED flag
|
||||
func initEnvironmentFlag(cmd *cobra.Command) {
|
||||
// To operate in the context of a bundle, all commands must take an "environment" parameter.
|
||||
cmd.PersistentFlags().StringP("environment", "e", "", "bundle environment to use (if applicable)")
|
||||
cmd.RegisterFlagCompletionFunc("environment", environmentCompletion)
|
||||
cmd.PersistentFlags().StringP("environment", "e", "", "bundle target to use (if applicable)")
|
||||
cmd.PersistentFlags().MarkDeprecated("environment", "use --target flag instead")
|
||||
cmd.RegisterFlagCompletionFunc("environment", targetCompletion)
|
||||
}
|
||||
|
|
|
@ -128,3 +128,27 @@ func TestBundleConfigureWithProfileFlagAndEnvVariable(t *testing.T) {
|
|||
b.WorkspaceClient()
|
||||
})
|
||||
}
|
||||
|
||||
func TestTargetFlagFull(t *testing.T) {
|
||||
cmd := emptyCommand(t)
|
||||
initTargetFlag(cmd)
|
||||
cmd.SetArgs([]string{"version", "--target", "development"})
|
||||
|
||||
ctx := context.Background()
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, cmd.Flag("target").Value.String(), "development")
|
||||
}
|
||||
|
||||
func TestTargetFlagShort(t *testing.T) {
|
||||
cmd := emptyCommand(t)
|
||||
initTargetFlag(cmd)
|
||||
cmd.SetArgs([]string{"version", "-t", "production"})
|
||||
|
||||
ctx := context.Background()
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, cmd.Flag("target").Value.String(), "production")
|
||||
}
|
||||
|
|
|
@ -36,6 +36,7 @@ func New() *cobra.Command {
|
|||
outputFlag := initOutputFlag(cmd)
|
||||
initProfileFlag(cmd)
|
||||
initEnvironmentFlag(cmd)
|
||||
initTargetFlag(cmd)
|
||||
|
||||
cmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {
|
||||
ctx := cmd.Context()
|
||||
|
|
|
@ -18,7 +18,7 @@ func TestSyncOptionsFromBundle(t *testing.T) {
|
|||
Path: tempDir,
|
||||
|
||||
Bundle: config.Bundle{
|
||||
Environment: "default",
|
||||
Target: "default",
|
||||
},
|
||||
|
||||
Workspace: config.Workspace{
|
||||
|
|
Loading…
Reference in New Issue