mirror of https://github.com/databricks/cli.git
Renamed `environments` to `targets` in bundle configuration (#670)
## Changes Renamed Environments to Targets in bundle.yml. The change is backward-compatible and customers can continue to use `environments` in the time being. ## Tests Added tests which checks that both `environments` and `targets` sections in bundle.yml works correctly
This commit is contained in:
parent
4694832534
commit
56dcd3f0a7
|
@ -117,10 +117,10 @@ func (b *Bundle) WorkspaceClient() *databricks.WorkspaceClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
// CacheDir returns directory to use for temporary files for this bundle.
|
// CacheDir returns directory to use for temporary files for this bundle.
|
||||||
// Scoped to the bundle's environment.
|
// Scoped to the bundle's target.
|
||||||
func (b *Bundle) CacheDir(paths ...string) (string, error) {
|
func (b *Bundle) CacheDir(paths ...string) (string, error) {
|
||||||
if b.Config.Bundle.Environment == "" {
|
if b.Config.Bundle.Target == "" {
|
||||||
panic("environment not set")
|
panic("target not set")
|
||||||
}
|
}
|
||||||
|
|
||||||
cacheDirName, exists := os.LookupEnv("DATABRICKS_BUNDLE_TMP")
|
cacheDirName, exists := os.LookupEnv("DATABRICKS_BUNDLE_TMP")
|
||||||
|
@ -138,8 +138,8 @@ func (b *Bundle) CacheDir(paths ...string) (string, error) {
|
||||||
// Fixed components of the result path.
|
// Fixed components of the result path.
|
||||||
parts := []string{
|
parts := []string{
|
||||||
cacheDirName,
|
cacheDirName,
|
||||||
// Scope with environment name.
|
// Scope with target name.
|
||||||
b.Config.Bundle.Environment,
|
b.Config.Bundle.Target,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Append dynamic components of the result path.
|
// Append dynamic components of the result path.
|
||||||
|
|
|
@ -31,16 +31,16 @@ func TestBundleCacheDir(t *testing.T) {
|
||||||
bundle, err := Load(context.Background(), projectDir)
|
bundle, err := Load(context.Background(), projectDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Artificially set environment.
|
// Artificially set target.
|
||||||
// This is otherwise done by [mutators.SelectEnvironment].
|
// This is otherwise done by [mutators.SelectTarget].
|
||||||
bundle.Config.Bundle.Environment = "default"
|
bundle.Config.Bundle.Target = "default"
|
||||||
|
|
||||||
// unset env variable in case it's set
|
// unset env variable in case it's set
|
||||||
t.Setenv("DATABRICKS_BUNDLE_TMP", "")
|
t.Setenv("DATABRICKS_BUNDLE_TMP", "")
|
||||||
|
|
||||||
cacheDir, err := bundle.CacheDir()
|
cacheDir, err := bundle.CacheDir()
|
||||||
|
|
||||||
// format is <CWD>/.databricks/bundle/<environment>
|
// format is <CWD>/.databricks/bundle/<target>
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.Equal(t, filepath.Join(projectDir, ".databricks", "bundle", "default"), cacheDir)
|
assert.Equal(t, filepath.Join(projectDir, ".databricks", "bundle", "default"), cacheDir)
|
||||||
}
|
}
|
||||||
|
@ -55,16 +55,16 @@ func TestBundleCacheDirOverride(t *testing.T) {
|
||||||
bundle, err := Load(context.Background(), projectDir)
|
bundle, err := Load(context.Background(), projectDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Artificially set environment.
|
// Artificially set target.
|
||||||
// This is otherwise done by [mutators.SelectEnvironment].
|
// This is otherwise done by [mutators.SelectTarget].
|
||||||
bundle.Config.Bundle.Environment = "default"
|
bundle.Config.Bundle.Target = "default"
|
||||||
|
|
||||||
// now we expect to use 'bundleTmpDir' instead of CWD/.databricks/bundle
|
// now we expect to use 'bundleTmpDir' instead of CWD/.databricks/bundle
|
||||||
t.Setenv("DATABRICKS_BUNDLE_TMP", bundleTmpDir)
|
t.Setenv("DATABRICKS_BUNDLE_TMP", bundleTmpDir)
|
||||||
|
|
||||||
cacheDir, err := bundle.CacheDir()
|
cacheDir, err := bundle.CacheDir()
|
||||||
|
|
||||||
// format is <DATABRICKS_BUNDLE_TMP>/<environment>
|
// format is <DATABRICKS_BUNDLE_TMP>/<target>
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.Equal(t, filepath.Join(bundleTmpDir, "default"), cacheDir)
|
assert.Equal(t, filepath.Join(bundleTmpDir, "default"), cacheDir)
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,7 +15,10 @@ type Bundle struct {
|
||||||
// Default warehouse to run SQL on.
|
// Default warehouse to run SQL on.
|
||||||
// DefaultWarehouse string `json:"default_warehouse,omitempty"`
|
// DefaultWarehouse string `json:"default_warehouse,omitempty"`
|
||||||
|
|
||||||
// Environment is set by the mutator that selects the environment.
|
// Target is set by the mutator that selects the target.
|
||||||
|
Target string `json:"target,omitempty" bundle:"readonly"`
|
||||||
|
|
||||||
|
// DEPRECATED. Left for backward compatibility with Target
|
||||||
Environment string `json:"environment,omitempty" bundle:"readonly"`
|
Environment string `json:"environment,omitempty" bundle:"readonly"`
|
||||||
|
|
||||||
// Terraform holds configuration related to Terraform.
|
// Terraform holds configuration related to Terraform.
|
||||||
|
@ -32,10 +35,10 @@ type Bundle struct {
|
||||||
// origin url. Automatically loaded by reading .git directory if not specified
|
// origin url. Automatically loaded by reading .git directory if not specified
|
||||||
Git Git `json:"git,omitempty"`
|
Git Git `json:"git,omitempty"`
|
||||||
|
|
||||||
// Determines the mode of the environment.
|
// Determines the mode of the target.
|
||||||
// For example, 'mode: development' can be used for deployments for
|
// For example, 'mode: development' can be used for deployments for
|
||||||
// development purposes.
|
// development purposes.
|
||||||
// Annotated readonly as this should be set at the environment level.
|
// Annotated readonly as this should be set at the target level.
|
||||||
Mode Mode `json:"mode,omitempty" bundle:"readonly"`
|
Mode Mode `json:"mode,omitempty" bundle:"readonly"`
|
||||||
|
|
||||||
// Overrides the compute used for jobs and other supported assets.
|
// Overrides the compute used for jobs and other supported assets.
|
||||||
|
|
|
@ -1,37 +0,0 @@
|
||||||
package mutator
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
|
||||||
"github.com/databricks/cli/bundle/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
type defineDefaultEnvironment struct {
|
|
||||||
name string
|
|
||||||
}
|
|
||||||
|
|
||||||
// DefineDefaultEnvironment adds an environment named "default"
|
|
||||||
// to the configuration if none have been defined.
|
|
||||||
func DefineDefaultEnvironment() bundle.Mutator {
|
|
||||||
return &defineDefaultEnvironment{
|
|
||||||
name: "default",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *defineDefaultEnvironment) Name() string {
|
|
||||||
return fmt.Sprintf("DefineDefaultEnvironment(%s)", m.name)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *defineDefaultEnvironment) Apply(_ context.Context, b *bundle.Bundle) error {
|
|
||||||
// Nothing to do if the configuration has at least 1 environment.
|
|
||||||
if len(b.Config.Environments) > 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Define default environment.
|
|
||||||
b.Config.Environments = make(map[string]*config.Environment)
|
|
||||||
b.Config.Environments[m.name] = &config.Environment{}
|
|
||||||
return nil
|
|
||||||
}
|
|
|
@ -0,0 +1,37 @@
|
||||||
|
package mutator
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
type defineDefaultTarget struct {
|
||||||
|
name string
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefineDefaultTarget adds a target named "default"
|
||||||
|
// to the configuration if none have been defined.
|
||||||
|
func DefineDefaultTarget() bundle.Mutator {
|
||||||
|
return &defineDefaultTarget{
|
||||||
|
name: "default",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *defineDefaultTarget) Name() string {
|
||||||
|
return fmt.Sprintf("DefineDefaultTarget(%s)", m.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *defineDefaultTarget) Apply(_ context.Context, b *bundle.Bundle) error {
|
||||||
|
// Nothing to do if the configuration has at least 1 target.
|
||||||
|
if len(b.Config.Targets) > 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Define default target.
|
||||||
|
b.Config.Targets = make(map[string]*config.Target)
|
||||||
|
b.Config.Targets[m.name] = &config.Target{}
|
||||||
|
return nil
|
||||||
|
}
|
|
@ -11,25 +11,25 @@ import (
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDefaultEnvironment(t *testing.T) {
|
func TestDefaultTarget(t *testing.T) {
|
||||||
bundle := &bundle.Bundle{}
|
bundle := &bundle.Bundle{}
|
||||||
err := mutator.DefineDefaultEnvironment().Apply(context.Background(), bundle)
|
err := mutator.DefineDefaultTarget().Apply(context.Background(), bundle)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
env, ok := bundle.Config.Environments["default"]
|
env, ok := bundle.Config.Targets["default"]
|
||||||
assert.True(t, ok)
|
assert.True(t, ok)
|
||||||
assert.Equal(t, &config.Environment{}, env)
|
assert.Equal(t, &config.Target{}, env)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDefaultEnvironmentAlreadySpecified(t *testing.T) {
|
func TestDefaultTargetAlreadySpecified(t *testing.T) {
|
||||||
bundle := &bundle.Bundle{
|
bundle := &bundle.Bundle{
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
Environments: map[string]*config.Environment{
|
Targets: map[string]*config.Target{
|
||||||
"development": {},
|
"development": {},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
err := mutator.DefineDefaultEnvironment().Apply(context.Background(), bundle)
|
err := mutator.DefineDefaultTarget().Apply(context.Background(), bundle)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
_, ok := bundle.Config.Environments["default"]
|
_, ok := bundle.Config.Targets["default"]
|
||||||
assert.False(t, ok)
|
assert.False(t, ok)
|
||||||
}
|
}
|
|
@ -27,14 +27,14 @@ func (m *defineDefaultWorkspaceRoot) Apply(ctx context.Context, b *bundle.Bundle
|
||||||
return fmt.Errorf("unable to define default workspace root: bundle name not defined")
|
return fmt.Errorf("unable to define default workspace root: bundle name not defined")
|
||||||
}
|
}
|
||||||
|
|
||||||
if b.Config.Bundle.Environment == "" {
|
if b.Config.Bundle.Target == "" {
|
||||||
return fmt.Errorf("unable to define default workspace root: bundle environment not selected")
|
return fmt.Errorf("unable to define default workspace root: bundle target not selected")
|
||||||
}
|
}
|
||||||
|
|
||||||
b.Config.Workspace.RootPath = fmt.Sprintf(
|
b.Config.Workspace.RootPath = fmt.Sprintf(
|
||||||
"~/.bundle/%s/%s",
|
"~/.bundle/%s/%s",
|
||||||
b.Config.Bundle.Name,
|
b.Config.Bundle.Name,
|
||||||
b.Config.Bundle.Environment,
|
b.Config.Bundle.Target,
|
||||||
)
|
)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -16,7 +16,7 @@ func TestDefaultWorkspaceRoot(t *testing.T) {
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
Bundle: config.Bundle{
|
Bundle: config.Bundle{
|
||||||
Name: "name",
|
Name: "name",
|
||||||
Environment: "environment",
|
Target: "environment",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,11 +7,11 @@ import (
|
||||||
func DefaultMutators() []bundle.Mutator {
|
func DefaultMutators() []bundle.Mutator {
|
||||||
return []bundle.Mutator{
|
return []bundle.Mutator{
|
||||||
ProcessRootIncludes(),
|
ProcessRootIncludes(),
|
||||||
DefineDefaultEnvironment(),
|
DefineDefaultTarget(),
|
||||||
LoadGitDetails(),
|
LoadGitDetails(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func DefaultMutatorsForEnvironment(env string) []bundle.Mutator {
|
func DefaultMutatorsForTarget(env string) []bundle.Mutator {
|
||||||
return append(DefaultMutators(), SelectEnvironment(env))
|
return append(DefaultMutators(), SelectTarget(env))
|
||||||
}
|
}
|
||||||
|
|
|
@ -35,7 +35,7 @@ func overrideJobCompute(j *resources.Job, compute string) {
|
||||||
func (m *overrideCompute) Apply(ctx context.Context, b *bundle.Bundle) error {
|
func (m *overrideCompute) Apply(ctx context.Context, b *bundle.Bundle) error {
|
||||||
if b.Config.Bundle.Mode != config.Development {
|
if b.Config.Bundle.Mode != config.Development {
|
||||||
if b.Config.Bundle.ComputeID != "" {
|
if b.Config.Bundle.ComputeID != "" {
|
||||||
return fmt.Errorf("cannot override compute for an environment that does not use 'mode: development'")
|
return fmt.Errorf("cannot override compute for an target that does not use 'mode: development'")
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,16 +13,16 @@ import (
|
||||||
"github.com/databricks/databricks-sdk-go/service/ml"
|
"github.com/databricks/databricks-sdk-go/service/ml"
|
||||||
)
|
)
|
||||||
|
|
||||||
type processEnvironmentMode struct{}
|
type processTargetMode struct{}
|
||||||
|
|
||||||
const developmentConcurrentRuns = 4
|
const developmentConcurrentRuns = 4
|
||||||
|
|
||||||
func ProcessEnvironmentMode() bundle.Mutator {
|
func ProcessTargetMode() bundle.Mutator {
|
||||||
return &processEnvironmentMode{}
|
return &processTargetMode{}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *processEnvironmentMode) Name() string {
|
func (m *processTargetMode) Name() string {
|
||||||
return "ProcessEnvironmentMode"
|
return "ProcessTargetMode"
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mark all resources as being for 'development' purposes, i.e.
|
// Mark all resources as being for 'development' purposes, i.e.
|
||||||
|
@ -110,14 +110,14 @@ func findIncorrectPath(b *bundle.Bundle, mode config.Mode) string {
|
||||||
|
|
||||||
func validateProductionMode(ctx context.Context, b *bundle.Bundle, isPrincipalUsed bool) error {
|
func validateProductionMode(ctx context.Context, b *bundle.Bundle, isPrincipalUsed bool) error {
|
||||||
if b.Config.Bundle.Git.Inferred {
|
if b.Config.Bundle.Git.Inferred {
|
||||||
env := b.Config.Bundle.Environment
|
env := b.Config.Bundle.Target
|
||||||
return fmt.Errorf("environment with 'mode: production' must specify an explicit 'environments.%s.git' configuration", env)
|
return fmt.Errorf("target with 'mode: production' must specify an explicit 'targets.%s.git' configuration", env)
|
||||||
}
|
}
|
||||||
|
|
||||||
r := b.Config.Resources
|
r := b.Config.Resources
|
||||||
for i := range r.Pipelines {
|
for i := range r.Pipelines {
|
||||||
if r.Pipelines[i].Development {
|
if r.Pipelines[i].Development {
|
||||||
return fmt.Errorf("environment with 'mode: production' cannot specify a pipeline with 'development: true'")
|
return fmt.Errorf("target with 'mode: production' cannot specify a pipeline with 'development: true'")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -125,7 +125,7 @@ func validateProductionMode(ctx context.Context, b *bundle.Bundle, isPrincipalUs
|
||||||
if path := findIncorrectPath(b, config.Production); path != "" {
|
if path := findIncorrectPath(b, config.Production); path != "" {
|
||||||
message := "%s must not contain the current username when using 'mode: production'"
|
message := "%s must not contain the current username when using 'mode: production'"
|
||||||
if path == "root_path" {
|
if path == "root_path" {
|
||||||
return fmt.Errorf(message+"\n tip: set workspace.root_path to a shared path such as /Shared/.bundle/${bundle.name}/${bundle.environment}", path)
|
return fmt.Errorf(message+"\n tip: set workspace.root_path to a shared path such as /Shared/.bundle/${bundle.name}/${bundle.target}", path)
|
||||||
} else {
|
} else {
|
||||||
return fmt.Errorf(message, path)
|
return fmt.Errorf(message, path)
|
||||||
}
|
}
|
||||||
|
@ -165,7 +165,7 @@ func isRunAsSet(r config.Resources) bool {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *processEnvironmentMode) Apply(ctx context.Context, b *bundle.Bundle) error {
|
func (m *processTargetMode) Apply(ctx context.Context, b *bundle.Bundle) error {
|
||||||
switch b.Config.Bundle.Mode {
|
switch b.Config.Bundle.Mode {
|
||||||
case config.Development:
|
case config.Development:
|
||||||
err := validateDevelopmentMode(b)
|
err := validateDevelopmentMode(b)
|
|
@ -58,10 +58,10 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProcessEnvironmentModeDevelopment(t *testing.T) {
|
func TestProcessTargetModeDevelopment(t *testing.T) {
|
||||||
bundle := mockBundle(config.Development)
|
bundle := mockBundle(config.Development)
|
||||||
|
|
||||||
m := ProcessEnvironmentMode()
|
m := ProcessTargetMode()
|
||||||
err := m.Apply(context.Background(), bundle)
|
err := m.Apply(context.Background(), bundle)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, "[dev lennart] job1", bundle.Config.Resources.Jobs["job1"].Name)
|
assert.Equal(t, "[dev lennart] job1", bundle.Config.Resources.Jobs["job1"].Name)
|
||||||
|
@ -73,10 +73,10 @@ func TestProcessEnvironmentModeDevelopment(t *testing.T) {
|
||||||
assert.True(t, bundle.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
|
assert.True(t, bundle.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProcessEnvironmentModeDefault(t *testing.T) {
|
func TestProcessTargetModeDefault(t *testing.T) {
|
||||||
bundle := mockBundle("")
|
bundle := mockBundle("")
|
||||||
|
|
||||||
m := ProcessEnvironmentMode()
|
m := ProcessTargetMode()
|
||||||
err := m.Apply(context.Background(), bundle)
|
err := m.Apply(context.Background(), bundle)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, "job1", bundle.Config.Resources.Jobs["job1"].Name)
|
assert.Equal(t, "job1", bundle.Config.Resources.Jobs["job1"].Name)
|
||||||
|
@ -84,7 +84,7 @@ func TestProcessEnvironmentModeDefault(t *testing.T) {
|
||||||
assert.False(t, bundle.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
|
assert.False(t, bundle.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProcessEnvironmentModeProduction(t *testing.T) {
|
func TestProcessTargetModeProduction(t *testing.T) {
|
||||||
bundle := mockBundle(config.Production)
|
bundle := mockBundle(config.Production)
|
||||||
|
|
||||||
err := validateProductionMode(context.Background(), bundle, false)
|
err := validateProductionMode(context.Background(), bundle, false)
|
||||||
|
@ -118,7 +118,7 @@ func TestProcessEnvironmentModeProduction(t *testing.T) {
|
||||||
assert.False(t, bundle.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
|
assert.False(t, bundle.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProcessEnvironmentModeProductionGit(t *testing.T) {
|
func TestProcessTargetModeProductionGit(t *testing.T) {
|
||||||
bundle := mockBundle(config.Production)
|
bundle := mockBundle(config.Production)
|
||||||
|
|
||||||
// Pretend the user didn't set Git configuration explicitly
|
// Pretend the user didn't set Git configuration explicitly
|
||||||
|
@ -129,10 +129,10 @@ func TestProcessEnvironmentModeProductionGit(t *testing.T) {
|
||||||
bundle.Config.Bundle.Git.Inferred = false
|
bundle.Config.Bundle.Git.Inferred = false
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProcessEnvironmentModeProductionOkForPrincipal(t *testing.T) {
|
func TestProcessTargetModeProductionOkForPrincipal(t *testing.T) {
|
||||||
bundle := mockBundle(config.Production)
|
bundle := mockBundle(config.Production)
|
||||||
|
|
||||||
// Our environment has all kinds of problems when not using service principals ...
|
// Our target has all kinds of problems when not using service principals ...
|
||||||
err := validateProductionMode(context.Background(), bundle, false)
|
err := validateProductionMode(context.Background(), bundle, false)
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
|
|
||||||
|
@ -152,7 +152,7 @@ func TestAllResourcesMocked(t *testing.T) {
|
||||||
assert.True(
|
assert.True(
|
||||||
t,
|
t,
|
||||||
!field.IsNil() && field.Len() > 0,
|
!field.IsNil() && field.Len() > 0,
|
||||||
"process_environment_mode should support '%s' (please add it to process_environment_mode.go and extend the test suite)",
|
"process_target_mode should support '%s' (please add it to process_target_mode.go and extend the test suite)",
|
||||||
resources.Type().Field(i).Name,
|
resources.Type().Field(i).Name,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
@ -164,7 +164,7 @@ func TestAllResourcesRenamed(t *testing.T) {
|
||||||
bundle := mockBundle(config.Development)
|
bundle := mockBundle(config.Development)
|
||||||
resources := reflect.ValueOf(bundle.Config.Resources)
|
resources := reflect.ValueOf(bundle.Config.Resources)
|
||||||
|
|
||||||
m := ProcessEnvironmentMode()
|
m := ProcessTargetMode()
|
||||||
err := m.Apply(context.Background(), bundle)
|
err := m.Apply(context.Background(), bundle)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
@ -179,7 +179,7 @@ func TestAllResourcesRenamed(t *testing.T) {
|
||||||
assert.True(
|
assert.True(
|
||||||
t,
|
t,
|
||||||
strings.Contains(nameField.String(), "dev"),
|
strings.Contains(nameField.String(), "dev"),
|
||||||
"process_environment_mode should rename '%s' in '%s'",
|
"process_target_mode should rename '%s' in '%s'",
|
||||||
key,
|
key,
|
||||||
resources.Type().Field(i).Name,
|
resources.Type().Field(i).Name,
|
||||||
)
|
)
|
|
@ -1,54 +0,0 @@
|
||||||
package mutator
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
|
||||||
"golang.org/x/exp/maps"
|
|
||||||
)
|
|
||||||
|
|
||||||
type selectDefaultEnvironment struct{}
|
|
||||||
|
|
||||||
// SelectDefaultEnvironment merges the default environment into the root configuration.
|
|
||||||
func SelectDefaultEnvironment() bundle.Mutator {
|
|
||||||
return &selectDefaultEnvironment{}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *selectDefaultEnvironment) Name() string {
|
|
||||||
return "SelectDefaultEnvironment"
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *selectDefaultEnvironment) Apply(ctx context.Context, b *bundle.Bundle) error {
|
|
||||||
if len(b.Config.Environments) == 0 {
|
|
||||||
return fmt.Errorf("no environments defined")
|
|
||||||
}
|
|
||||||
|
|
||||||
// One environment means there's only one default.
|
|
||||||
names := maps.Keys(b.Config.Environments)
|
|
||||||
if len(names) == 1 {
|
|
||||||
return SelectEnvironment(names[0]).Apply(ctx, b)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Multiple environments means we look for the `default` flag.
|
|
||||||
var defaults []string
|
|
||||||
for name, env := range b.Config.Environments {
|
|
||||||
if env != nil && env.Default {
|
|
||||||
defaults = append(defaults, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// It is invalid to have multiple environments with the `default` flag set.
|
|
||||||
if len(defaults) > 1 {
|
|
||||||
return fmt.Errorf("multiple environments are marked as default (%s)", strings.Join(defaults, ", "))
|
|
||||||
}
|
|
||||||
|
|
||||||
// If no environment has the `default` flag set, ask the user to specify one.
|
|
||||||
if len(defaults) == 0 {
|
|
||||||
return fmt.Errorf("please specify environment")
|
|
||||||
}
|
|
||||||
|
|
||||||
// One default remaining.
|
|
||||||
return SelectEnvironment(defaults[0]).Apply(ctx, b)
|
|
||||||
}
|
|
|
@ -1,90 +0,0 @@
|
||||||
package mutator_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
|
||||||
"github.com/databricks/cli/bundle/config"
|
|
||||||
"github.com/databricks/cli/bundle/config/mutator"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestSelectDefaultEnvironmentNoEnvironments(t *testing.T) {
|
|
||||||
bundle := &bundle.Bundle{
|
|
||||||
Config: config.Root{
|
|
||||||
Environments: map[string]*config.Environment{},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
err := mutator.SelectDefaultEnvironment().Apply(context.Background(), bundle)
|
|
||||||
assert.ErrorContains(t, err, "no environments defined")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSelectDefaultEnvironmentSingleEnvironments(t *testing.T) {
|
|
||||||
bundle := &bundle.Bundle{
|
|
||||||
Config: config.Root{
|
|
||||||
Environments: map[string]*config.Environment{
|
|
||||||
"foo": {},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
err := mutator.SelectDefaultEnvironment().Apply(context.Background(), bundle)
|
|
||||||
assert.NoError(t, err)
|
|
||||||
assert.Equal(t, "foo", bundle.Config.Bundle.Environment)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSelectDefaultEnvironmentNoDefaults(t *testing.T) {
|
|
||||||
bundle := &bundle.Bundle{
|
|
||||||
Config: config.Root{
|
|
||||||
Environments: map[string]*config.Environment{
|
|
||||||
"foo": {},
|
|
||||||
"bar": {},
|
|
||||||
"qux": {},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
err := mutator.SelectDefaultEnvironment().Apply(context.Background(), bundle)
|
|
||||||
assert.ErrorContains(t, err, "please specify environment")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSelectDefaultEnvironmentNoDefaultsWithNil(t *testing.T) {
|
|
||||||
bundle := &bundle.Bundle{
|
|
||||||
Config: config.Root{
|
|
||||||
Environments: map[string]*config.Environment{
|
|
||||||
"foo": nil,
|
|
||||||
"bar": nil,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
err := mutator.SelectDefaultEnvironment().Apply(context.Background(), bundle)
|
|
||||||
assert.ErrorContains(t, err, "please specify environment")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSelectDefaultEnvironmentMultipleDefaults(t *testing.T) {
|
|
||||||
bundle := &bundle.Bundle{
|
|
||||||
Config: config.Root{
|
|
||||||
Environments: map[string]*config.Environment{
|
|
||||||
"foo": {Default: true},
|
|
||||||
"bar": {Default: true},
|
|
||||||
"qux": {Default: true},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
err := mutator.SelectDefaultEnvironment().Apply(context.Background(), bundle)
|
|
||||||
assert.ErrorContains(t, err, "multiple environments are marked as default")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSelectDefaultEnvironmentSingleDefault(t *testing.T) {
|
|
||||||
bundle := &bundle.Bundle{
|
|
||||||
Config: config.Root{
|
|
||||||
Environments: map[string]*config.Environment{
|
|
||||||
"foo": {},
|
|
||||||
"bar": {Default: true},
|
|
||||||
"qux": {},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
err := mutator.SelectDefaultEnvironment().Apply(context.Background(), bundle)
|
|
||||||
assert.NoError(t, err)
|
|
||||||
assert.Equal(t, "bar", bundle.Config.Bundle.Environment)
|
|
||||||
}
|
|
|
@ -0,0 +1,54 @@
|
||||||
|
package mutator
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"golang.org/x/exp/maps"
|
||||||
|
)
|
||||||
|
|
||||||
|
type selectDefaultTarget struct{}
|
||||||
|
|
||||||
|
// SelectDefaultTarget merges the default target into the root configuration.
|
||||||
|
func SelectDefaultTarget() bundle.Mutator {
|
||||||
|
return &selectDefaultTarget{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *selectDefaultTarget) Name() string {
|
||||||
|
return "SelectDefaultTarget"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *selectDefaultTarget) Apply(ctx context.Context, b *bundle.Bundle) error {
|
||||||
|
if len(b.Config.Targets) == 0 {
|
||||||
|
return fmt.Errorf("no targets defined")
|
||||||
|
}
|
||||||
|
|
||||||
|
// One target means there's only one default.
|
||||||
|
names := maps.Keys(b.Config.Targets)
|
||||||
|
if len(names) == 1 {
|
||||||
|
return SelectTarget(names[0]).Apply(ctx, b)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Multiple targets means we look for the `default` flag.
|
||||||
|
var defaults []string
|
||||||
|
for name, env := range b.Config.Targets {
|
||||||
|
if env != nil && env.Default {
|
||||||
|
defaults = append(defaults, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// It is invalid to have multiple targets with the `default` flag set.
|
||||||
|
if len(defaults) > 1 {
|
||||||
|
return fmt.Errorf("multiple targets are marked as default (%s)", strings.Join(defaults, ", "))
|
||||||
|
}
|
||||||
|
|
||||||
|
// If no target has the `default` flag set, ask the user to specify one.
|
||||||
|
if len(defaults) == 0 {
|
||||||
|
return fmt.Errorf("please specify target")
|
||||||
|
}
|
||||||
|
|
||||||
|
// One default remaining.
|
||||||
|
return SelectTarget(defaults[0]).Apply(ctx, b)
|
||||||
|
}
|
|
@ -0,0 +1,90 @@
|
||||||
|
package mutator_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config"
|
||||||
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSelectDefaultTargetNoTargets(t *testing.T) {
|
||||||
|
bundle := &bundle.Bundle{
|
||||||
|
Config: config.Root{
|
||||||
|
Targets: map[string]*config.Target{},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
err := mutator.SelectDefaultTarget().Apply(context.Background(), bundle)
|
||||||
|
assert.ErrorContains(t, err, "no targets defined")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelectDefaultTargetSingleTargets(t *testing.T) {
|
||||||
|
bundle := &bundle.Bundle{
|
||||||
|
Config: config.Root{
|
||||||
|
Targets: map[string]*config.Target{
|
||||||
|
"foo": {},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
err := mutator.SelectDefaultTarget().Apply(context.Background(), bundle)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, "foo", bundle.Config.Bundle.Target)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelectDefaultTargetNoDefaults(t *testing.T) {
|
||||||
|
bundle := &bundle.Bundle{
|
||||||
|
Config: config.Root{
|
||||||
|
Targets: map[string]*config.Target{
|
||||||
|
"foo": {},
|
||||||
|
"bar": {},
|
||||||
|
"qux": {},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
err := mutator.SelectDefaultTarget().Apply(context.Background(), bundle)
|
||||||
|
assert.ErrorContains(t, err, "please specify target")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelectDefaultTargetNoDefaultsWithNil(t *testing.T) {
|
||||||
|
bundle := &bundle.Bundle{
|
||||||
|
Config: config.Root{
|
||||||
|
Targets: map[string]*config.Target{
|
||||||
|
"foo": nil,
|
||||||
|
"bar": nil,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
err := mutator.SelectDefaultTarget().Apply(context.Background(), bundle)
|
||||||
|
assert.ErrorContains(t, err, "please specify target")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelectDefaultTargetMultipleDefaults(t *testing.T) {
|
||||||
|
bundle := &bundle.Bundle{
|
||||||
|
Config: config.Root{
|
||||||
|
Targets: map[string]*config.Target{
|
||||||
|
"foo": {Default: true},
|
||||||
|
"bar": {Default: true},
|
||||||
|
"qux": {Default: true},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
err := mutator.SelectDefaultTarget().Apply(context.Background(), bundle)
|
||||||
|
assert.ErrorContains(t, err, "multiple targets are marked as default")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelectDefaultTargetSingleDefault(t *testing.T) {
|
||||||
|
bundle := &bundle.Bundle{
|
||||||
|
Config: config.Root{
|
||||||
|
Targets: map[string]*config.Target{
|
||||||
|
"foo": {},
|
||||||
|
"bar": {Default: true},
|
||||||
|
"qux": {},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
err := mutator.SelectDefaultTarget().Apply(context.Background(), bundle)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, "bar", bundle.Config.Bundle.Target)
|
||||||
|
}
|
|
@ -1,48 +0,0 @@
|
||||||
package mutator
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
|
||||||
)
|
|
||||||
|
|
||||||
type selectEnvironment struct {
|
|
||||||
name string
|
|
||||||
}
|
|
||||||
|
|
||||||
// SelectEnvironment merges the specified environment into the root configuration.
|
|
||||||
func SelectEnvironment(name string) bundle.Mutator {
|
|
||||||
return &selectEnvironment{
|
|
||||||
name: name,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *selectEnvironment) Name() string {
|
|
||||||
return fmt.Sprintf("SelectEnvironment(%s)", m.name)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *selectEnvironment) Apply(_ context.Context, b *bundle.Bundle) error {
|
|
||||||
if b.Config.Environments == nil {
|
|
||||||
return fmt.Errorf("no environments defined")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get specified environment
|
|
||||||
env, ok := b.Config.Environments[m.name]
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("%s: no such environment", m.name)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Merge specified environment into root configuration structure.
|
|
||||||
err := b.Config.MergeEnvironment(env)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Store specified environment in configuration for reference.
|
|
||||||
b.Config.Bundle.Environment = m.name
|
|
||||||
|
|
||||||
// Clear environments after loading.
|
|
||||||
b.Config.Environments = nil
|
|
||||||
return nil
|
|
||||||
}
|
|
|
@ -0,0 +1,54 @@
|
||||||
|
package mutator
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
)
|
||||||
|
|
||||||
|
type selectTarget struct {
|
||||||
|
name string
|
||||||
|
}
|
||||||
|
|
||||||
|
// SelectTarget merges the specified target into the root configuration.
|
||||||
|
func SelectTarget(name string) bundle.Mutator {
|
||||||
|
return &selectTarget{
|
||||||
|
name: name,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *selectTarget) Name() string {
|
||||||
|
return fmt.Sprintf("SelectTarget(%s)", m.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *selectTarget) Apply(_ context.Context, b *bundle.Bundle) error {
|
||||||
|
if b.Config.Targets == nil {
|
||||||
|
return fmt.Errorf("no targets defined")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get specified target
|
||||||
|
target, ok := b.Config.Targets[m.name]
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("%s: no such target", m.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Merge specified target into root configuration structure.
|
||||||
|
err := b.Config.MergeTargetOverrides(target)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store specified target in configuration for reference.
|
||||||
|
b.Config.Bundle.Target = m.name
|
||||||
|
|
||||||
|
// We do this for backward compatibility.
|
||||||
|
// TODO: remove when Environments section is not supported anymore.
|
||||||
|
b.Config.Bundle.Environment = b.Config.Bundle.Target
|
||||||
|
|
||||||
|
// Clear targets after loading.
|
||||||
|
b.Config.Targets = nil
|
||||||
|
b.Config.Environments = nil
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
|
@ -11,13 +11,13 @@ import (
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestSelectEnvironment(t *testing.T) {
|
func TestSelectTarget(t *testing.T) {
|
||||||
bundle := &bundle.Bundle{
|
bundle := &bundle.Bundle{
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
Workspace: config.Workspace{
|
Workspace: config.Workspace{
|
||||||
Host: "foo",
|
Host: "foo",
|
||||||
},
|
},
|
||||||
Environments: map[string]*config.Environment{
|
Targets: map[string]*config.Target{
|
||||||
"default": {
|
"default": {
|
||||||
Workspace: &config.Workspace{
|
Workspace: &config.Workspace{
|
||||||
Host: "bar",
|
Host: "bar",
|
||||||
|
@ -26,19 +26,19 @@ func TestSelectEnvironment(t *testing.T) {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
err := mutator.SelectEnvironment("default").Apply(context.Background(), bundle)
|
err := mutator.SelectTarget("default").Apply(context.Background(), bundle)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, "bar", bundle.Config.Workspace.Host)
|
assert.Equal(t, "bar", bundle.Config.Workspace.Host)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSelectEnvironmentNotFound(t *testing.T) {
|
func TestSelectTargetNotFound(t *testing.T) {
|
||||||
bundle := &bundle.Bundle{
|
bundle := &bundle.Bundle{
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
Environments: map[string]*config.Environment{
|
Targets: map[string]*config.Target{
|
||||||
"default": {},
|
"default": {},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
err := mutator.SelectEnvironment("doesnt-exist").Apply(context.Background(), bundle)
|
err := mutator.SelectTarget("doesnt-exist").Apply(context.Background(), bundle)
|
||||||
require.Error(t, err, "no environments defined")
|
require.Error(t, err, "no targets defined")
|
||||||
}
|
}
|
|
@ -115,7 +115,7 @@ func (r *Resources) SetConfigFilePath(path string) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// MergeJobClusters iterates over all jobs and merges their job clusters.
|
// MergeJobClusters iterates over all jobs and merges their job clusters.
|
||||||
// This is called after applying the environment overrides.
|
// This is called after applying the target overrides.
|
||||||
func (r *Resources) MergeJobClusters() error {
|
func (r *Resources) MergeJobClusters() error {
|
||||||
for _, job := range r.Jobs {
|
for _, job := range r.Jobs {
|
||||||
if err := job.MergeJobClusters(); err != nil {
|
if err := job.MergeJobClusters(); err != nil {
|
||||||
|
|
|
@ -22,7 +22,7 @@ func (j *Job) MergeJobClusters() error {
|
||||||
keys := make(map[string]*jobs.JobCluster)
|
keys := make(map[string]*jobs.JobCluster)
|
||||||
output := make([]jobs.JobCluster, 0, len(j.JobClusters))
|
output := make([]jobs.JobCluster, 0, len(j.JobClusters))
|
||||||
|
|
||||||
// Environment overrides are always appended, so we can iterate in natural order to
|
// Target overrides are always appended, so we can iterate in natural order to
|
||||||
// first find the base definition, and merge instances we encounter later.
|
// first find the base definition, and merge instances we encounter later.
|
||||||
for i := range j.JobClusters {
|
for i := range j.JobClusters {
|
||||||
key := j.JobClusters[i].JobClusterKey
|
key := j.JobClusters[i].JobClusterKey
|
||||||
|
|
|
@ -69,11 +69,14 @@ type Root struct {
|
||||||
// to deploy in this bundle (e.g. jobs, pipelines, etc.).
|
// to deploy in this bundle (e.g. jobs, pipelines, etc.).
|
||||||
Resources Resources `json:"resources,omitempty"`
|
Resources Resources `json:"resources,omitempty"`
|
||||||
|
|
||||||
// Environments can be used to differentiate settings and resources between
|
// Targets can be used to differentiate settings and resources between
|
||||||
// bundle deployment environments (e.g. development, staging, production).
|
// bundle deployment targets (e.g. development, staging, production).
|
||||||
// If not specified, the code below initializes this field with a
|
// If not specified, the code below initializes this field with a
|
||||||
// single default-initialized environment called "default".
|
// single default-initialized target called "default".
|
||||||
Environments map[string]*Environment `json:"environments,omitempty"`
|
Targets map[string]*Target `json:"targets,omitempty"`
|
||||||
|
|
||||||
|
// DEPRECATED. Left for backward compatibility with Targets
|
||||||
|
Environments map[string]*Target `json:"environments,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func Load(path string) (*Root, error) {
|
func Load(path string) (*Root, error) {
|
||||||
|
@ -103,8 +106,8 @@ func Load(path string) (*Root, error) {
|
||||||
// was loaded from in configuration leafs that require it.
|
// was loaded from in configuration leafs that require it.
|
||||||
func (r *Root) SetConfigFilePath(path string) {
|
func (r *Root) SetConfigFilePath(path string) {
|
||||||
r.Resources.SetConfigFilePath(path)
|
r.Resources.SetConfigFilePath(path)
|
||||||
if r.Environments != nil {
|
if r.Targets != nil {
|
||||||
for _, env := range r.Environments {
|
for _, env := range r.Targets {
|
||||||
if env == nil {
|
if env == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
@ -148,6 +151,15 @@ func (r *Root) Load(path string) error {
|
||||||
return fmt.Errorf("failed to load %s: %w", path, err)
|
return fmt.Errorf("failed to load %s: %w", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if r.Environments != nil && r.Targets != nil {
|
||||||
|
return fmt.Errorf("both 'environments' and 'targets' are specified, only 'targets' should be used: %s", path)
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.Environments != nil {
|
||||||
|
//TODO: add a command line notice that this is a deprecated option.
|
||||||
|
r.Targets = r.Environments
|
||||||
|
}
|
||||||
|
|
||||||
r.Path = filepath.Dir(path)
|
r.Path = filepath.Dir(path)
|
||||||
r.SetConfigFilePath(path)
|
r.SetConfigFilePath(path)
|
||||||
|
|
||||||
|
@ -169,37 +181,37 @@ func (r *Root) Merge(other *Root) error {
|
||||||
return mergo.Merge(r, other, mergo.WithOverride)
|
return mergo.Merge(r, other, mergo.WithOverride)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Root) MergeEnvironment(env *Environment) error {
|
func (r *Root) MergeTargetOverrides(target *Target) error {
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
// Environment may be nil if it's empty.
|
// Target may be nil if it's empty.
|
||||||
if env == nil {
|
if target == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if env.Bundle != nil {
|
if target.Bundle != nil {
|
||||||
err = mergo.Merge(&r.Bundle, env.Bundle, mergo.WithOverride)
|
err = mergo.Merge(&r.Bundle, target.Bundle, mergo.WithOverride)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if env.Workspace != nil {
|
if target.Workspace != nil {
|
||||||
err = mergo.Merge(&r.Workspace, env.Workspace, mergo.WithOverride)
|
err = mergo.Merge(&r.Workspace, target.Workspace, mergo.WithOverride)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if env.Artifacts != nil {
|
if target.Artifacts != nil {
|
||||||
err = mergo.Merge(&r.Artifacts, env.Artifacts, mergo.WithOverride, mergo.WithAppendSlice)
|
err = mergo.Merge(&r.Artifacts, target.Artifacts, mergo.WithOverride, mergo.WithAppendSlice)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if env.Resources != nil {
|
if target.Resources != nil {
|
||||||
err = mergo.Merge(&r.Resources, env.Resources, mergo.WithOverride, mergo.WithAppendSlice)
|
err = mergo.Merge(&r.Resources, target.Resources, mergo.WithOverride, mergo.WithAppendSlice)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -210,8 +222,8 @@ func (r *Root) MergeEnvironment(env *Environment) error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if env.Variables != nil {
|
if target.Variables != nil {
|
||||||
for k, v := range env.Variables {
|
for k, v := range target.Variables {
|
||||||
variable, ok := r.Variables[k]
|
variable, ok := r.Variables[k]
|
||||||
if !ok {
|
if !ok {
|
||||||
return fmt.Errorf("variable %s is not defined but is assigned a value", k)
|
return fmt.Errorf("variable %s is not defined but is assigned a value", k)
|
||||||
|
@ -222,24 +234,24 @@ func (r *Root) MergeEnvironment(env *Environment) error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if env.Mode != "" {
|
if target.Mode != "" {
|
||||||
r.Bundle.Mode = env.Mode
|
r.Bundle.Mode = target.Mode
|
||||||
}
|
}
|
||||||
|
|
||||||
if env.ComputeID != "" {
|
if target.ComputeID != "" {
|
||||||
r.Bundle.ComputeID = env.ComputeID
|
r.Bundle.ComputeID = target.ComputeID
|
||||||
}
|
}
|
||||||
|
|
||||||
git := &r.Bundle.Git
|
git := &r.Bundle.Git
|
||||||
if env.Git.Branch != "" {
|
if target.Git.Branch != "" {
|
||||||
git.Branch = env.Git.Branch
|
git.Branch = target.Git.Branch
|
||||||
git.Inferred = false
|
git.Inferred = false
|
||||||
}
|
}
|
||||||
if env.Git.Commit != "" {
|
if target.Git.Commit != "" {
|
||||||
git.Commit = env.Git.Commit
|
git.Commit = target.Git.Commit
|
||||||
}
|
}
|
||||||
if env.Git.OriginURL != "" {
|
if target.Git.OriginURL != "" {
|
||||||
git.OriginURL = env.Git.OriginURL
|
git.OriginURL = target.Git.OriginURL
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|
|
@ -57,7 +57,7 @@ func TestRootMergeStruct(t *testing.T) {
|
||||||
func TestRootMergeMap(t *testing.T) {
|
func TestRootMergeMap(t *testing.T) {
|
||||||
root := &Root{
|
root := &Root{
|
||||||
Path: "path",
|
Path: "path",
|
||||||
Environments: map[string]*Environment{
|
Targets: map[string]*Target{
|
||||||
"development": {
|
"development": {
|
||||||
Workspace: &Workspace{
|
Workspace: &Workspace{
|
||||||
Host: "foo",
|
Host: "foo",
|
||||||
|
@ -68,7 +68,7 @@ func TestRootMergeMap(t *testing.T) {
|
||||||
}
|
}
|
||||||
other := &Root{
|
other := &Root{
|
||||||
Path: "path",
|
Path: "path",
|
||||||
Environments: map[string]*Environment{
|
Targets: map[string]*Target{
|
||||||
"development": {
|
"development": {
|
||||||
Workspace: &Workspace{
|
Workspace: &Workspace{
|
||||||
Host: "bar",
|
Host: "bar",
|
||||||
|
@ -77,7 +77,7 @@ func TestRootMergeMap(t *testing.T) {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
assert.NoError(t, root.Merge(other))
|
assert.NoError(t, root.Merge(other))
|
||||||
assert.Equal(t, &Workspace{Host: "bar", Profile: "profile"}, root.Environments["development"].Workspace)
|
assert.Equal(t, &Workspace{Host: "bar", Profile: "profile"}, root.Targets["development"].Workspace)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDuplicateIdOnLoadReturnsError(t *testing.T) {
|
func TestDuplicateIdOnLoadReturnsError(t *testing.T) {
|
||||||
|
@ -159,12 +159,12 @@ func TestInitializeVariablesUndefinedVariables(t *testing.T) {
|
||||||
assert.ErrorContains(t, err, "variable bar has not been defined")
|
assert.ErrorContains(t, err, "variable bar has not been defined")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRootMergeEnvironmentWithMode(t *testing.T) {
|
func TestRootMergeTargetOverridesWithMode(t *testing.T) {
|
||||||
root := &Root{
|
root := &Root{
|
||||||
Bundle: Bundle{},
|
Bundle: Bundle{},
|
||||||
}
|
}
|
||||||
env := &Environment{Mode: Development}
|
env := &Target{Mode: Development}
|
||||||
require.NoError(t, root.MergeEnvironment(env))
|
require.NoError(t, root.MergeTargetOverrides(env))
|
||||||
assert.Equal(t, Development, root.Bundle.Mode)
|
assert.Equal(t, Development, root.Bundle.Mode)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -2,14 +2,14 @@ package config
|
||||||
|
|
||||||
type Mode string
|
type Mode string
|
||||||
|
|
||||||
// Environment defines overrides for a single environment.
|
// Target defines overrides for a single target.
|
||||||
// This structure is recursively merged into the root configuration.
|
// This structure is recursively merged into the root configuration.
|
||||||
type Environment struct {
|
type Target struct {
|
||||||
// Default marks that this environment must be used if one isn't specified
|
// Default marks that this target must be used if one isn't specified
|
||||||
// by the user (through environment variable or command line argument).
|
// by the user (through target variable or command line argument).
|
||||||
Default bool `json:"default,omitempty"`
|
Default bool `json:"default,omitempty"`
|
||||||
|
|
||||||
// Determines the mode of the environment.
|
// Determines the mode of the target.
|
||||||
// For example, 'mode: development' can be used for deployments for
|
// For example, 'mode: development' can be used for deployments for
|
||||||
// development purposes.
|
// development purposes.
|
||||||
Mode Mode `json:"mode,omitempty"`
|
Mode Mode `json:"mode,omitempty"`
|
||||||
|
@ -27,7 +27,7 @@ type Environment struct {
|
||||||
|
|
||||||
// Override default values for defined variables
|
// Override default values for defined variables
|
||||||
// Does not permit defining new variables or redefining existing ones
|
// Does not permit defining new variables or redefining existing ones
|
||||||
// in the scope of an environment
|
// in the scope of an target
|
||||||
Variables map[string]string `json:"variables,omitempty"`
|
Variables map[string]string `json:"variables,omitempty"`
|
||||||
|
|
||||||
Git Git `json:"git,omitempty"`
|
Git Git `json:"git,omitempty"`
|
|
@ -18,7 +18,7 @@ type Variable struct {
|
||||||
// resolved in the following priority order (from highest to lowest)
|
// resolved in the following priority order (from highest to lowest)
|
||||||
//
|
//
|
||||||
// 1. Command line flag. For example: `--var="foo=bar"`
|
// 1. Command line flag. For example: `--var="foo=bar"`
|
||||||
// 2. Environment variable. eg: BUNDLE_VAR_foo=bar
|
// 2. Target variable. eg: BUNDLE_VAR_foo=bar
|
||||||
// 3. Default value as defined in the applicable environments block
|
// 3. Default value as defined in the applicable environments block
|
||||||
// 4. Default value defined in variable definition
|
// 4. Default value defined in variable definition
|
||||||
// 5. Throw error, since if no default value is defined, then the variable
|
// 5. Throw error, since if no default value is defined, then the variable
|
||||||
|
|
|
@ -45,7 +45,7 @@ type Workspace struct {
|
||||||
CurrentUser *User `json:"current_user,omitempty" bundle:"readonly"`
|
CurrentUser *User `json:"current_user,omitempty" bundle:"readonly"`
|
||||||
|
|
||||||
// Remote workspace base path for deployment state, for artifacts, as synchronization target.
|
// Remote workspace base path for deployment state, for artifacts, as synchronization target.
|
||||||
// This defaults to "~/.bundle/${bundle.name}/${bundle.environment}" where "~" expands to
|
// This defaults to "~/.bundle/${bundle.name}/${bundle.target}" where "~" expands to
|
||||||
// the current user's home directory in the workspace (e.g. `/Users/jane@doe.com`).
|
// the current user's home directory in the workspace (e.g. `/Users/jane@doe.com`).
|
||||||
RootPath string `json:"root_path,omitempty"`
|
RootPath string `json:"root_path,omitempty"`
|
||||||
|
|
||||||
|
|
|
@ -31,7 +31,7 @@ func TestInitEnvironmentVariables(t *testing.T) {
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
Path: t.TempDir(),
|
Path: t.TempDir(),
|
||||||
Bundle: config.Bundle{
|
Bundle: config.Bundle{
|
||||||
Environment: "whatever",
|
Target: "whatever",
|
||||||
Terraform: &config.Terraform{
|
Terraform: &config.Terraform{
|
||||||
ExecPath: "terraform",
|
ExecPath: "terraform",
|
||||||
},
|
},
|
||||||
|
@ -58,7 +58,7 @@ func TestSetTempDirEnvVarsForUnixWithTmpDirSet(t *testing.T) {
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
Path: t.TempDir(),
|
Path: t.TempDir(),
|
||||||
Bundle: config.Bundle{
|
Bundle: config.Bundle{
|
||||||
Environment: "whatever",
|
Target: "whatever",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -86,7 +86,7 @@ func TestSetTempDirEnvVarsForUnixWithTmpDirNotSet(t *testing.T) {
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
Path: t.TempDir(),
|
Path: t.TempDir(),
|
||||||
Bundle: config.Bundle{
|
Bundle: config.Bundle{
|
||||||
Environment: "whatever",
|
Target: "whatever",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -112,7 +112,7 @@ func TestSetTempDirEnvVarsForWindowWithAllTmpDirEnvVarsSet(t *testing.T) {
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
Path: t.TempDir(),
|
Path: t.TempDir(),
|
||||||
Bundle: config.Bundle{
|
Bundle: config.Bundle{
|
||||||
Environment: "whatever",
|
Target: "whatever",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -142,7 +142,7 @@ func TestSetTempDirEnvVarsForWindowWithUserProfileAndTempSet(t *testing.T) {
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
Path: t.TempDir(),
|
Path: t.TempDir(),
|
||||||
Bundle: config.Bundle{
|
Bundle: config.Bundle{
|
||||||
Environment: "whatever",
|
Target: "whatever",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -172,7 +172,7 @@ func TestSetTempDirEnvVarsForWindowWithUserProfileSet(t *testing.T) {
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
Path: t.TempDir(),
|
Path: t.TempDir(),
|
||||||
Bundle: config.Bundle{
|
Bundle: config.Bundle{
|
||||||
Environment: "whatever",
|
Target: "whatever",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -202,7 +202,7 @@ func TestSetTempDirEnvVarsForWindowsWithoutAnyTempDirEnvVarsSet(t *testing.T) {
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
Path: t.TempDir(),
|
Path: t.TempDir(),
|
||||||
Bundle: config.Bundle{
|
Bundle: config.Bundle{
|
||||||
Environment: "whatever",
|
Target: "whatever",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -230,7 +230,7 @@ func TestSetProxyEnvVars(t *testing.T) {
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
Path: t.TempDir(),
|
Path: t.TempDir(),
|
||||||
Bundle: config.Bundle{
|
Bundle: config.Bundle{
|
||||||
Environment: "whatever",
|
Target: "whatever",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,7 +20,7 @@ func TestLoadWithNoState(t *testing.T) {
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
Path: t.TempDir(),
|
Path: t.TempDir(),
|
||||||
Bundle: config.Bundle{
|
Bundle: config.Bundle{
|
||||||
Environment: "whatever",
|
Target: "whatever",
|
||||||
Terraform: &config.Terraform{
|
Terraform: &config.Terraform{
|
||||||
ExecPath: "terraform",
|
ExecPath: "terraform",
|
||||||
},
|
},
|
||||||
|
|
|
@ -26,7 +26,7 @@ func Initialize() bundle.Mutator {
|
||||||
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
|
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
|
||||||
),
|
),
|
||||||
mutator.OverrideCompute(),
|
mutator.OverrideCompute(),
|
||||||
mutator.ProcessEnvironmentMode(),
|
mutator.ProcessTargetMode(),
|
||||||
mutator.TranslatePaths(),
|
mutator.TranslatePaths(),
|
||||||
terraform.Initialize(),
|
terraform.Initialize(),
|
||||||
},
|
},
|
||||||
|
|
|
@ -3,7 +3,7 @@
|
||||||
`docs/bundle_descriptions.json` contains both autogenerated as well as manually written
|
`docs/bundle_descriptions.json` contains both autogenerated as well as manually written
|
||||||
descriptions for the json schema. Specifically
|
descriptions for the json schema. Specifically
|
||||||
1. `resources` : almost all descriptions are autogenerated from the OpenAPI spec
|
1. `resources` : almost all descriptions are autogenerated from the OpenAPI spec
|
||||||
2. `environments` : almost all descriptions are copied over from root level entities (eg: `bundle`, `artifacts`)
|
2. `targets` : almost all descriptions are copied over from root level entities (eg: `bundle`, `artifacts`)
|
||||||
3. `bundle` : manually editted
|
3. `bundle` : manually editted
|
||||||
4. `include` : manually editted
|
4. `include` : manually editted
|
||||||
5. `workspace` : manually editted
|
5. `workspace` : manually editted
|
||||||
|
@ -17,7 +17,7 @@ These descriptions are rendered in the inline documentation in an IDE
|
||||||
`databricks bundle schema --only-docs > ~/databricks/bundle/schema/docs/bundle_descriptions.json`
|
`databricks bundle schema --only-docs > ~/databricks/bundle/schema/docs/bundle_descriptions.json`
|
||||||
2. Manually edit bundle_descriptions.json to add your descriptions
|
2. Manually edit bundle_descriptions.json to add your descriptions
|
||||||
3. Build again to embed the new `bundle_descriptions.json` into the binary (`go build`)
|
3. Build again to embed the new `bundle_descriptions.json` into the binary (`go build`)
|
||||||
4. Again run `databricks bundle schema --only-docs > ~/databricks/bundle/schema/docs/bundle_descriptions.json` to copy over any applicable descriptions to `environments`
|
4. Again run `databricks bundle schema --only-docs > ~/databricks/bundle/schema/docs/bundle_descriptions.json` to copy over any applicable descriptions to `targets`
|
||||||
5. push to repo
|
5. push to repo
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -52,20 +52,20 @@ func BundleDocs(openapiSpecPath string) (*Docs, error) {
|
||||||
}
|
}
|
||||||
docs.Properties["resources"] = schemaToDocs(resourceSchema)
|
docs.Properties["resources"] = schemaToDocs(resourceSchema)
|
||||||
}
|
}
|
||||||
docs.refreshEnvironmentsDocs()
|
docs.refreshTargetsDocs()
|
||||||
return docs, nil
|
return docs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (docs *Docs) refreshEnvironmentsDocs() error {
|
func (docs *Docs) refreshTargetsDocs() error {
|
||||||
environmentsDocs, ok := docs.Properties["environments"]
|
targetsDocs, ok := docs.Properties["targets"]
|
||||||
if !ok || environmentsDocs.AdditionalProperties == nil ||
|
if !ok || targetsDocs.AdditionalProperties == nil ||
|
||||||
environmentsDocs.AdditionalProperties.Properties == nil {
|
targetsDocs.AdditionalProperties.Properties == nil {
|
||||||
return fmt.Errorf("invalid environments descriptions")
|
return fmt.Errorf("invalid targets descriptions")
|
||||||
}
|
}
|
||||||
environmentProperties := environmentsDocs.AdditionalProperties.Properties
|
targetProperties := targetsDocs.AdditionalProperties.Properties
|
||||||
propertiesToCopy := []string{"artifacts", "bundle", "resources", "workspace"}
|
propertiesToCopy := []string{"artifacts", "bundle", "resources", "workspace"}
|
||||||
for _, p := range propertiesToCopy {
|
for _, p := range propertiesToCopy {
|
||||||
environmentProperties[p] = docs.Properties[p]
|
targetProperties[p] = docs.Properties[p]
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -36,7 +36,7 @@
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"environments": {
|
"targets": {
|
||||||
"description": "",
|
"description": "",
|
||||||
"additionalproperties": {
|
"additionalproperties": {
|
||||||
"description": "",
|
"description": "",
|
||||||
|
@ -1827,7 +1827,7 @@
|
||||||
"description": "Connection profile to use. By default profiles are specified in ~/.databrickscfg."
|
"description": "Connection profile to use. By default profiles are specified in ~/.databrickscfg."
|
||||||
},
|
},
|
||||||
"root_path": {
|
"root_path": {
|
||||||
"description": "The base location for synchronizing files, artifacts and state. Defaults to `/Users/jane@doe.com/.bundle/${bundle.name}/${bundle.environment}`"
|
"description": "The base location for synchronizing files, artifacts and state. Defaults to `/Users/jane@doe.com/.bundle/${bundle.name}/${bundle.target}`"
|
||||||
},
|
},
|
||||||
"state_path": {
|
"state_path": {
|
||||||
"description": "The remote path to synchronize bundle state to. This defaults to `${workspace.root}/state`"
|
"description": "The remote path to synchronize bundle state to. This defaults to `${workspace.root}/state`"
|
||||||
|
@ -3591,7 +3591,7 @@
|
||||||
"description": "Connection profile to use. By default profiles are specified in ~/.databrickscfg."
|
"description": "Connection profile to use. By default profiles are specified in ~/.databrickscfg."
|
||||||
},
|
},
|
||||||
"root_path": {
|
"root_path": {
|
||||||
"description": "The base location for synchronizing files, artifacts and state. Defaults to `/Users/jane@doe.com/.bundle/${bundle.name}/${bundle.environment}`"
|
"description": "The base location for synchronizing files, artifacts and state. Defaults to `/Users/jane@doe.com/.bundle/${bundle.name}/${bundle.target}`"
|
||||||
},
|
},
|
||||||
"state_path": {
|
"state_path": {
|
||||||
"description": "The remote path to synchronize bundle state to. This defaults to `${workspace.root}/state`"
|
"description": "The remote path to synchronize bundle state to. This defaults to `${workspace.root}/state`"
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
bundle:
|
bundle:
|
||||||
name: autoload git config test
|
name: autoload git config test
|
||||||
|
|
||||||
environments:
|
targets:
|
||||||
development:
|
development:
|
||||||
default: true
|
default: true
|
||||||
|
|
||||||
|
|
|
@ -1,5 +0,0 @@
|
||||||
bundle:
|
|
||||||
name: environment_empty
|
|
||||||
|
|
||||||
environments:
|
|
||||||
development:
|
|
|
@ -1,12 +0,0 @@
|
||||||
package config_tests
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestEnvironmentEmpty(t *testing.T) {
|
|
||||||
b := loadEnvironment(t, "./environment_empty", "development")
|
|
||||||
assert.Equal(t, "development", b.Config.Bundle.Environment)
|
|
||||||
}
|
|
|
@ -0,0 +1,20 @@
|
||||||
|
package config_tests
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGitAutoLoadWithEnvironment(t *testing.T) {
|
||||||
|
b := load(t, "./environments_autoload_git")
|
||||||
|
assert.True(t, b.Config.Bundle.Git.Inferred)
|
||||||
|
assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGitManuallySetBranchWithEnvironment(t *testing.T) {
|
||||||
|
b := loadTarget(t, "./environments_autoload_git", "production")
|
||||||
|
assert.False(t, b.Config.Bundle.Git.Inferred)
|
||||||
|
assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
|
||||||
|
assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli")
|
||||||
|
}
|
|
@ -7,17 +7,17 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestEnvironmentOverridesWorkspaceDev(t *testing.T) {
|
func TestEnvironmentOverridesWorkspaceDev(t *testing.T) {
|
||||||
b := loadEnvironment(t, "./environment_overrides/workspace", "development")
|
b := loadTarget(t, "./environment_overrides/workspace", "development")
|
||||||
assert.Equal(t, "https://development.acme.cloud.databricks.com/", b.Config.Workspace.Host)
|
assert.Equal(t, "https://development.acme.cloud.databricks.com/", b.Config.Workspace.Host)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEnvironmentOverridesWorkspaceStaging(t *testing.T) {
|
func TestEnvironmentOverridesWorkspaceStaging(t *testing.T) {
|
||||||
b := loadEnvironment(t, "./environment_overrides/workspace", "staging")
|
b := loadTarget(t, "./environment_overrides/workspace", "staging")
|
||||||
assert.Equal(t, "https://staging.acme.cloud.databricks.com/", b.Config.Workspace.Host)
|
assert.Equal(t, "https://staging.acme.cloud.databricks.com/", b.Config.Workspace.Host)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEnvironmentOverridesResourcesDev(t *testing.T) {
|
func TestEnvironmentOverridesResourcesDev(t *testing.T) {
|
||||||
b := loadEnvironment(t, "./environment_overrides/resources", "development")
|
b := loadTarget(t, "./environment_overrides/resources", "development")
|
||||||
assert.Equal(t, "base job", b.Config.Resources.Jobs["job1"].Name)
|
assert.Equal(t, "base job", b.Config.Resources.Jobs["job1"].Name)
|
||||||
|
|
||||||
// Base values are preserved in the development environment.
|
// Base values are preserved in the development environment.
|
||||||
|
@ -26,7 +26,7 @@ func TestEnvironmentOverridesResourcesDev(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEnvironmentOverridesResourcesStaging(t *testing.T) {
|
func TestEnvironmentOverridesResourcesStaging(t *testing.T) {
|
||||||
b := loadEnvironment(t, "./environment_overrides/resources", "staging")
|
b := loadTarget(t, "./environment_overrides/resources", "staging")
|
||||||
assert.Equal(t, "staging job", b.Config.Resources.Jobs["job1"].Name)
|
assert.Equal(t, "staging job", b.Config.Resources.Jobs["job1"].Name)
|
||||||
|
|
||||||
// Overrides are only applied if they are not zero-valued.
|
// Overrides are only applied if they are not zero-valued.
|
||||||
|
|
|
@ -0,0 +1,11 @@
|
||||||
|
bundle:
|
||||||
|
name: autoload git config test
|
||||||
|
|
||||||
|
environments:
|
||||||
|
development:
|
||||||
|
default: true
|
||||||
|
|
||||||
|
production:
|
||||||
|
# production can only be deployed from the 'main' branch
|
||||||
|
git:
|
||||||
|
branch: main
|
|
@ -0,0 +1,44 @@
|
||||||
|
resources:
|
||||||
|
pipelines:
|
||||||
|
nyc_taxi_pipeline:
|
||||||
|
name: "nyc taxi loader"
|
||||||
|
libraries:
|
||||||
|
- notebook:
|
||||||
|
path: ./dlt/nyc_taxi_loader
|
||||||
|
|
||||||
|
environments:
|
||||||
|
development:
|
||||||
|
mode: development
|
||||||
|
resources:
|
||||||
|
pipelines:
|
||||||
|
nyc_taxi_pipeline:
|
||||||
|
target: nyc_taxi_development
|
||||||
|
development: true
|
||||||
|
|
||||||
|
staging:
|
||||||
|
resources:
|
||||||
|
pipelines:
|
||||||
|
nyc_taxi_pipeline:
|
||||||
|
target: nyc_taxi_staging
|
||||||
|
development: false
|
||||||
|
|
||||||
|
production:
|
||||||
|
mode: production
|
||||||
|
resources:
|
||||||
|
pipelines:
|
||||||
|
nyc_taxi_pipeline:
|
||||||
|
target: nyc_taxi_production
|
||||||
|
development: false
|
||||||
|
photon: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
pipeline_schedule:
|
||||||
|
name: Daily refresh of production pipeline
|
||||||
|
|
||||||
|
schedule:
|
||||||
|
quartz_cron_expression: 6 6 11 * * ?
|
||||||
|
timezone_id: UTC
|
||||||
|
|
||||||
|
tasks:
|
||||||
|
- pipeline_task:
|
||||||
|
pipeline_id: "to be interpolated"
|
|
@ -0,0 +1,56 @@
|
||||||
|
package config_tests
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle/config"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestJobAndPipelineDevelopmentWithEnvironment(t *testing.T) {
|
||||||
|
b := loadTarget(t, "./environments_job_and_pipeline", "development")
|
||||||
|
assert.Len(t, b.Config.Resources.Jobs, 0)
|
||||||
|
assert.Len(t, b.Config.Resources.Pipelines, 1)
|
||||||
|
|
||||||
|
p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"]
|
||||||
|
assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath))
|
||||||
|
assert.Equal(t, b.Config.Bundle.Mode, config.Development)
|
||||||
|
assert.True(t, p.Development)
|
||||||
|
require.Len(t, p.Libraries, 1)
|
||||||
|
assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path)
|
||||||
|
assert.Equal(t, "nyc_taxi_development", p.Target)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestJobAndPipelineStagingWithEnvironment(t *testing.T) {
|
||||||
|
b := loadTarget(t, "./environments_job_and_pipeline", "staging")
|
||||||
|
assert.Len(t, b.Config.Resources.Jobs, 0)
|
||||||
|
assert.Len(t, b.Config.Resources.Pipelines, 1)
|
||||||
|
|
||||||
|
p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"]
|
||||||
|
assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath))
|
||||||
|
assert.False(t, p.Development)
|
||||||
|
require.Len(t, p.Libraries, 1)
|
||||||
|
assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path)
|
||||||
|
assert.Equal(t, "nyc_taxi_staging", p.Target)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestJobAndPipelineProductionWithEnvironment(t *testing.T) {
|
||||||
|
b := loadTarget(t, "./environments_job_and_pipeline", "production")
|
||||||
|
assert.Len(t, b.Config.Resources.Jobs, 1)
|
||||||
|
assert.Len(t, b.Config.Resources.Pipelines, 1)
|
||||||
|
|
||||||
|
p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"]
|
||||||
|
assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath))
|
||||||
|
assert.False(t, p.Development)
|
||||||
|
require.Len(t, p.Libraries, 1)
|
||||||
|
assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path)
|
||||||
|
assert.Equal(t, "nyc_taxi_production", p.Target)
|
||||||
|
|
||||||
|
j := b.Config.Resources.Jobs["pipeline_schedule"]
|
||||||
|
assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(j.ConfigFilePath))
|
||||||
|
assert.Equal(t, "Daily refresh of production pipeline", j.Name)
|
||||||
|
require.Len(t, j.Tasks, 1)
|
||||||
|
assert.NotEmpty(t, j.Tasks[0].PipelineTask.PipelineId)
|
||||||
|
}
|
|
@ -0,0 +1,35 @@
|
||||||
|
bundle:
|
||||||
|
name: override_job_cluster
|
||||||
|
|
||||||
|
workspace:
|
||||||
|
host: https://acme.cloud.databricks.com/
|
||||||
|
|
||||||
|
resources:
|
||||||
|
jobs:
|
||||||
|
foo:
|
||||||
|
name: job
|
||||||
|
job_clusters:
|
||||||
|
- job_cluster_key: key
|
||||||
|
new_cluster:
|
||||||
|
spark_version: 13.3.x-scala2.12
|
||||||
|
|
||||||
|
environments:
|
||||||
|
development:
|
||||||
|
resources:
|
||||||
|
jobs:
|
||||||
|
foo:
|
||||||
|
job_clusters:
|
||||||
|
- job_cluster_key: key
|
||||||
|
new_cluster:
|
||||||
|
node_type_id: i3.xlarge
|
||||||
|
num_workers: 1
|
||||||
|
|
||||||
|
staging:
|
||||||
|
resources:
|
||||||
|
jobs:
|
||||||
|
foo:
|
||||||
|
job_clusters:
|
||||||
|
- job_cluster_key: key
|
||||||
|
new_cluster:
|
||||||
|
node_type_id: i3.2xlarge
|
||||||
|
num_workers: 4
|
|
@ -0,0 +1,29 @@
|
||||||
|
package config_tests
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestOverrideJobClusterDevWithEnvironment(t *testing.T) {
|
||||||
|
b := loadTarget(t, "./environments_override_job_cluster", "development")
|
||||||
|
assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name)
|
||||||
|
assert.Len(t, b.Config.Resources.Jobs["foo"].JobClusters, 1)
|
||||||
|
|
||||||
|
c := b.Config.Resources.Jobs["foo"].JobClusters[0]
|
||||||
|
assert.Equal(t, "13.3.x-scala2.12", c.NewCluster.SparkVersion)
|
||||||
|
assert.Equal(t, "i3.xlarge", c.NewCluster.NodeTypeId)
|
||||||
|
assert.Equal(t, 1, c.NewCluster.NumWorkers)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOverrideJobClusterStagingWithEnvironment(t *testing.T) {
|
||||||
|
b := loadTarget(t, "./environments_override_job_cluster", "staging")
|
||||||
|
assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name)
|
||||||
|
assert.Len(t, b.Config.Resources.Jobs["foo"].JobClusters, 1)
|
||||||
|
|
||||||
|
c := b.Config.Resources.Jobs["foo"].JobClusters[0]
|
||||||
|
assert.Equal(t, "13.3.x-scala2.12", c.NewCluster.SparkVersion)
|
||||||
|
assert.Equal(t, "i3.2xlarge", c.NewCluster.NodeTypeId)
|
||||||
|
assert.Equal(t, 4, c.NewCluster.NumWorkers)
|
||||||
|
}
|
|
@ -17,7 +17,7 @@ func TestGitAutoLoad(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGitManuallySetBranch(t *testing.T) {
|
func TestGitManuallySetBranch(t *testing.T) {
|
||||||
b := loadEnvironment(t, "./autoload_git", "production")
|
b := loadTarget(t, "./autoload_git", "production")
|
||||||
assert.False(t, b.Config.Bundle.Git.Inferred)
|
assert.False(t, b.Config.Bundle.Git.Inferred)
|
||||||
assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
|
assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
|
||||||
assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli")
|
assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli")
|
||||||
|
|
|
@ -0,0 +1,14 @@
|
||||||
|
bundle:
|
||||||
|
name: foo ${workspace.profile}
|
||||||
|
|
||||||
|
workspace:
|
||||||
|
profile: bar
|
||||||
|
|
||||||
|
targets:
|
||||||
|
development:
|
||||||
|
default: true
|
||||||
|
|
||||||
|
resources:
|
||||||
|
jobs:
|
||||||
|
my_job:
|
||||||
|
name: "${bundle.name} | ${workspace.profile} | ${bundle.environment} | ${bundle.target}"
|
|
@ -20,3 +20,15 @@ func TestInterpolation(t *testing.T) {
|
||||||
assert.Equal(t, "foo bar", b.Config.Bundle.Name)
|
assert.Equal(t, "foo bar", b.Config.Bundle.Name)
|
||||||
assert.Equal(t, "foo bar | bar", b.Config.Resources.Jobs["my_job"].Name)
|
assert.Equal(t, "foo bar | bar", b.Config.Resources.Jobs["my_job"].Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestInterpolationWithTarget(t *testing.T) {
|
||||||
|
b := loadTarget(t, "./interpolation_target", "development")
|
||||||
|
err := bundle.Apply(context.Background(), b, interpolation.Interpolate(
|
||||||
|
interpolation.IncludeLookupsInPath("bundle"),
|
||||||
|
interpolation.IncludeLookupsInPath("workspace"),
|
||||||
|
))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "foo bar", b.Config.Bundle.Name)
|
||||||
|
assert.Equal(t, "foo bar | bar | development | development", b.Config.Resources.Jobs["my_job"].Name)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
|
@ -6,7 +6,7 @@ resources:
|
||||||
- notebook:
|
- notebook:
|
||||||
path: ./dlt/nyc_taxi_loader
|
path: ./dlt/nyc_taxi_loader
|
||||||
|
|
||||||
environments:
|
targets:
|
||||||
development:
|
development:
|
||||||
mode: development
|
mode: development
|
||||||
resources:
|
resources:
|
||||||
|
|
|
@ -10,7 +10,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestJobAndPipelineDevelopment(t *testing.T) {
|
func TestJobAndPipelineDevelopment(t *testing.T) {
|
||||||
b := loadEnvironment(t, "./job_and_pipeline", "development")
|
b := loadTarget(t, "./job_and_pipeline", "development")
|
||||||
assert.Len(t, b.Config.Resources.Jobs, 0)
|
assert.Len(t, b.Config.Resources.Jobs, 0)
|
||||||
assert.Len(t, b.Config.Resources.Pipelines, 1)
|
assert.Len(t, b.Config.Resources.Pipelines, 1)
|
||||||
|
|
||||||
|
@ -24,7 +24,7 @@ func TestJobAndPipelineDevelopment(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestJobAndPipelineStaging(t *testing.T) {
|
func TestJobAndPipelineStaging(t *testing.T) {
|
||||||
b := loadEnvironment(t, "./job_and_pipeline", "staging")
|
b := loadTarget(t, "./job_and_pipeline", "staging")
|
||||||
assert.Len(t, b.Config.Resources.Jobs, 0)
|
assert.Len(t, b.Config.Resources.Jobs, 0)
|
||||||
assert.Len(t, b.Config.Resources.Pipelines, 1)
|
assert.Len(t, b.Config.Resources.Pipelines, 1)
|
||||||
|
|
||||||
|
@ -37,7 +37,7 @@ func TestJobAndPipelineStaging(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestJobAndPipelineProduction(t *testing.T) {
|
func TestJobAndPipelineProduction(t *testing.T) {
|
||||||
b := loadEnvironment(t, "./job_and_pipeline", "production")
|
b := loadTarget(t, "./job_and_pipeline", "production")
|
||||||
assert.Len(t, b.Config.Resources.Jobs, 1)
|
assert.Len(t, b.Config.Resources.Jobs, 1)
|
||||||
assert.Len(t, b.Config.Resources.Pipelines, 1)
|
assert.Len(t, b.Config.Resources.Pipelines, 1)
|
||||||
|
|
||||||
|
|
|
@ -18,9 +18,9 @@ func load(t *testing.T, path string) *bundle.Bundle {
|
||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadEnvironment(t *testing.T, path, env string) *bundle.Bundle {
|
func loadTarget(t *testing.T, path, env string) *bundle.Bundle {
|
||||||
b := load(t, path)
|
b := load(t, path)
|
||||||
err := bundle.Apply(context.Background(), b, mutator.SelectEnvironment(env))
|
err := bundle.Apply(context.Background(), b, mutator.SelectTarget(env))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,7 +13,7 @@ resources:
|
||||||
new_cluster:
|
new_cluster:
|
||||||
spark_version: 13.3.x-scala2.12
|
spark_version: 13.3.x-scala2.12
|
||||||
|
|
||||||
environments:
|
targets:
|
||||||
development:
|
development:
|
||||||
resources:
|
resources:
|
||||||
jobs:
|
jobs:
|
||||||
|
|
|
@ -7,7 +7,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestOverrideJobClusterDev(t *testing.T) {
|
func TestOverrideJobClusterDev(t *testing.T) {
|
||||||
b := loadEnvironment(t, "./override_job_cluster", "development")
|
b := loadTarget(t, "./override_job_cluster", "development")
|
||||||
assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name)
|
assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name)
|
||||||
assert.Len(t, b.Config.Resources.Jobs["foo"].JobClusters, 1)
|
assert.Len(t, b.Config.Resources.Jobs["foo"].JobClusters, 1)
|
||||||
|
|
||||||
|
@ -18,7 +18,7 @@ func TestOverrideJobClusterDev(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOverrideJobClusterStaging(t *testing.T) {
|
func TestOverrideJobClusterStaging(t *testing.T) {
|
||||||
b := loadEnvironment(t, "./override_job_cluster", "staging")
|
b := loadTarget(t, "./override_job_cluster", "staging")
|
||||||
assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name)
|
assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name)
|
||||||
assert.Len(t, b.Config.Resources.Jobs["foo"].JobClusters, 1)
|
assert.Len(t, b.Config.Resources.Jobs["foo"].JobClusters, 1)
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,5 @@
|
||||||
|
bundle:
|
||||||
|
name: target_empty
|
||||||
|
|
||||||
|
targets:
|
||||||
|
development:
|
|
@ -0,0 +1,12 @@
|
||||||
|
package config_tests
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTargetEmpty(t *testing.T) {
|
||||||
|
b := loadTarget(t, "./target_empty", "development")
|
||||||
|
assert.Equal(t, "development", b.Config.Bundle.Target)
|
||||||
|
}
|
|
@ -0,0 +1,20 @@
|
||||||
|
bundle:
|
||||||
|
name: environment_overrides
|
||||||
|
|
||||||
|
workspace:
|
||||||
|
host: https://acme.cloud.databricks.com/
|
||||||
|
|
||||||
|
resources:
|
||||||
|
jobs:
|
||||||
|
job1:
|
||||||
|
name: "base job"
|
||||||
|
|
||||||
|
targets:
|
||||||
|
development:
|
||||||
|
default: true
|
||||||
|
|
||||||
|
staging:
|
||||||
|
resources:
|
||||||
|
jobs:
|
||||||
|
job1:
|
||||||
|
name: "staging job"
|
|
@ -0,0 +1,14 @@
|
||||||
|
bundle:
|
||||||
|
name: environment_overrides
|
||||||
|
|
||||||
|
workspace:
|
||||||
|
host: https://acme.cloud.databricks.com/
|
||||||
|
|
||||||
|
targets:
|
||||||
|
development:
|
||||||
|
workspace:
|
||||||
|
host: https://development.acme.cloud.databricks.com/
|
||||||
|
|
||||||
|
staging:
|
||||||
|
workspace:
|
||||||
|
host: https://staging.acme.cloud.databricks.com/
|
|
@ -0,0 +1,27 @@
|
||||||
|
package config_tests
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTargetOverridesWorkspaceDev(t *testing.T) {
|
||||||
|
b := loadTarget(t, "./target_overrides/workspace", "development")
|
||||||
|
assert.Equal(t, "https://development.acme.cloud.databricks.com/", b.Config.Workspace.Host)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTargetOverridesWorkspaceStaging(t *testing.T) {
|
||||||
|
b := loadTarget(t, "./target_overrides/workspace", "staging")
|
||||||
|
assert.Equal(t, "https://staging.acme.cloud.databricks.com/", b.Config.Workspace.Host)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTargetOverridesResourcesDev(t *testing.T) {
|
||||||
|
b := loadTarget(t, "./target_overrides/resources", "development")
|
||||||
|
assert.Equal(t, "base job", b.Config.Resources.Jobs["job1"].Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTargetOverridesResourcesStaging(t *testing.T) {
|
||||||
|
b := loadTarget(t, "./target_overrides/resources", "staging")
|
||||||
|
assert.Equal(t, "staging job", b.Config.Resources.Jobs["job1"].Name)
|
||||||
|
}
|
|
@ -12,7 +12,7 @@ bundle:
|
||||||
workspace:
|
workspace:
|
||||||
profile: ${var.a} ${var.b}
|
profile: ${var.a} ${var.b}
|
||||||
|
|
||||||
environments:
|
targets:
|
||||||
env-with-single-variable-override:
|
env-with-single-variable-override:
|
||||||
variables:
|
variables:
|
||||||
b: dev-b
|
b: dev-b
|
||||||
|
|
|
@ -34,10 +34,10 @@ func TestVariablesLoadingFailsWhenRequiredVariableIsNotSpecified(t *testing.T) {
|
||||||
assert.ErrorContains(t, err, "no value assigned to required variable b. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_b environment variable")
|
assert.ErrorContains(t, err, "no value assigned to required variable b. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_b environment variable")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestVariablesEnvironmentsBlockOverride(t *testing.T) {
|
func TestVariablesTargetsBlockOverride(t *testing.T) {
|
||||||
b := load(t, "./variables/env_overrides")
|
b := load(t, "./variables/env_overrides")
|
||||||
err := bundle.Apply(context.Background(), b, bundle.Seq(
|
err := bundle.Apply(context.Background(), b, bundle.Seq(
|
||||||
mutator.SelectEnvironment("env-with-single-variable-override"),
|
mutator.SelectTarget("env-with-single-variable-override"),
|
||||||
mutator.SetVariables(),
|
mutator.SetVariables(),
|
||||||
interpolation.Interpolate(
|
interpolation.Interpolate(
|
||||||
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
|
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
|
||||||
|
@ -46,10 +46,10 @@ func TestVariablesEnvironmentsBlockOverride(t *testing.T) {
|
||||||
assert.Equal(t, "default-a dev-b", b.Config.Workspace.Profile)
|
assert.Equal(t, "default-a dev-b", b.Config.Workspace.Profile)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestVariablesEnvironmentsBlockOverrideForMultipleVariables(t *testing.T) {
|
func TestVariablesTargetsBlockOverrideForMultipleVariables(t *testing.T) {
|
||||||
b := load(t, "./variables/env_overrides")
|
b := load(t, "./variables/env_overrides")
|
||||||
err := bundle.Apply(context.Background(), b, bundle.Seq(
|
err := bundle.Apply(context.Background(), b, bundle.Seq(
|
||||||
mutator.SelectEnvironment("env-with-two-variable-overrides"),
|
mutator.SelectTarget("env-with-two-variable-overrides"),
|
||||||
mutator.SetVariables(),
|
mutator.SetVariables(),
|
||||||
interpolation.Interpolate(
|
interpolation.Interpolate(
|
||||||
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
|
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
|
||||||
|
@ -58,11 +58,11 @@ func TestVariablesEnvironmentsBlockOverrideForMultipleVariables(t *testing.T) {
|
||||||
assert.Equal(t, "prod-a prod-b", b.Config.Workspace.Profile)
|
assert.Equal(t, "prod-a prod-b", b.Config.Workspace.Profile)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestVariablesEnvironmentsBlockOverrideWithProcessEnvVars(t *testing.T) {
|
func TestVariablesTargetsBlockOverrideWithProcessEnvVars(t *testing.T) {
|
||||||
t.Setenv("BUNDLE_VAR_b", "env-var-b")
|
t.Setenv("BUNDLE_VAR_b", "env-var-b")
|
||||||
b := load(t, "./variables/env_overrides")
|
b := load(t, "./variables/env_overrides")
|
||||||
err := bundle.Apply(context.Background(), b, bundle.Seq(
|
err := bundle.Apply(context.Background(), b, bundle.Seq(
|
||||||
mutator.SelectEnvironment("env-with-two-variable-overrides"),
|
mutator.SelectTarget("env-with-two-variable-overrides"),
|
||||||
mutator.SetVariables(),
|
mutator.SetVariables(),
|
||||||
interpolation.Interpolate(
|
interpolation.Interpolate(
|
||||||
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
|
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
|
||||||
|
@ -71,10 +71,10 @@ func TestVariablesEnvironmentsBlockOverrideWithProcessEnvVars(t *testing.T) {
|
||||||
assert.Equal(t, "prod-a env-var-b", b.Config.Workspace.Profile)
|
assert.Equal(t, "prod-a env-var-b", b.Config.Workspace.Profile)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestVariablesEnvironmentsBlockOverrideWithMissingVariables(t *testing.T) {
|
func TestVariablesTargetsBlockOverrideWithMissingVariables(t *testing.T) {
|
||||||
b := load(t, "./variables/env_overrides")
|
b := load(t, "./variables/env_overrides")
|
||||||
err := bundle.Apply(context.Background(), b, bundle.Seq(
|
err := bundle.Apply(context.Background(), b, bundle.Seq(
|
||||||
mutator.SelectEnvironment("env-missing-a-required-variable-assignment"),
|
mutator.SelectTarget("env-missing-a-required-variable-assignment"),
|
||||||
mutator.SetVariables(),
|
mutator.SetVariables(),
|
||||||
interpolation.Interpolate(
|
interpolation.Interpolate(
|
||||||
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
|
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
|
||||||
|
@ -82,10 +82,10 @@ func TestVariablesEnvironmentsBlockOverrideWithMissingVariables(t *testing.T) {
|
||||||
assert.ErrorContains(t, err, "no value assigned to required variable b. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_b environment variable")
|
assert.ErrorContains(t, err, "no value assigned to required variable b. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_b environment variable")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestVariablesEnvironmentsBlockOverrideWithUndefinedVariables(t *testing.T) {
|
func TestVariablesTargetsBlockOverrideWithUndefinedVariables(t *testing.T) {
|
||||||
b := load(t, "./variables/env_overrides")
|
b := load(t, "./variables/env_overrides")
|
||||||
err := bundle.Apply(context.Background(), b, bundle.Seq(
|
err := bundle.Apply(context.Background(), b, bundle.Seq(
|
||||||
mutator.SelectEnvironment("env-using-an-undefined-variable"),
|
mutator.SelectTarget("env-using-an-undefined-variable"),
|
||||||
mutator.SetVariables(),
|
mutator.SetVariables(),
|
||||||
interpolation.Interpolate(
|
interpolation.Interpolate(
|
||||||
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
|
interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
|
||||||
|
|
|
@ -7,7 +7,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func ConfigureBundleWithVariables(cmd *cobra.Command, args []string) error {
|
func ConfigureBundleWithVariables(cmd *cobra.Command, args []string) error {
|
||||||
// Load bundle config and apply environment
|
// Load bundle config and apply target
|
||||||
err := root.MustConfigureBundle(cmd, args)
|
err := root.MustConfigureBundle(cmd, args)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|
|
@ -131,7 +131,7 @@ func newConfigureCommand() *cobra.Command {
|
||||||
|
|
||||||
// Include token flag for compatibility with the legacy CLI.
|
// Include token flag for compatibility with the legacy CLI.
|
||||||
// It doesn't actually do anything because we always use PATs.
|
// It doesn't actually do anything because we always use PATs.
|
||||||
cmd.Flags().BoolP("token", "t", true, "Configure using Databricks Personal Access Token")
|
cmd.Flags().Bool("token", true, "Configure using Databricks Personal Access Token")
|
||||||
cmd.Flags().MarkHidden("token")
|
cmd.Flags().MarkHidden("token")
|
||||||
|
|
||||||
cmd.RunE = func(cmd *cobra.Command, args []string) error {
|
cmd.RunE = func(cmd *cobra.Command, args []string) error {
|
||||||
|
|
|
@ -11,11 +11,12 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
const envName = "DATABRICKS_BUNDLE_ENV"
|
const envName = "DATABRICKS_BUNDLE_ENV"
|
||||||
|
const targetName = "DATABRICKS_BUNDLE_TARGET"
|
||||||
|
|
||||||
// getEnvironment returns the name of the environment to operate in.
|
// getTarget returns the name of the target to operate in.
|
||||||
func getEnvironment(cmd *cobra.Command) (value string) {
|
func getTarget(cmd *cobra.Command) (value string) {
|
||||||
// The command line flag takes precedence.
|
// The command line flag takes precedence.
|
||||||
flag := cmd.Flag("environment")
|
flag := cmd.Flag("target")
|
||||||
if flag != nil {
|
if flag != nil {
|
||||||
value = flag.Value.String()
|
value = flag.Value.String()
|
||||||
if value != "" {
|
if value != "" {
|
||||||
|
@ -23,8 +24,23 @@ func getEnvironment(cmd *cobra.Command) (value string) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
oldFlag := cmd.Flag("environment")
|
||||||
|
if oldFlag != nil {
|
||||||
|
value = flag.Value.String()
|
||||||
|
if value != "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// If it's not set, use the environment variable.
|
// If it's not set, use the environment variable.
|
||||||
return os.Getenv(envName)
|
target := os.Getenv(targetName)
|
||||||
|
// If target env is not set with a new variable, try to check for old variable name
|
||||||
|
// TODO: remove when environments section is not supported anymore
|
||||||
|
if target == "" {
|
||||||
|
target = os.Getenv(envName)
|
||||||
|
}
|
||||||
|
|
||||||
|
return target
|
||||||
}
|
}
|
||||||
|
|
||||||
func getProfile(cmd *cobra.Command) (value string) {
|
func getProfile(cmd *cobra.Command) (value string) {
|
||||||
|
@ -80,11 +96,11 @@ func configureBundle(cmd *cobra.Command, args []string, load func(ctx context.Co
|
||||||
}
|
}
|
||||||
|
|
||||||
var m bundle.Mutator
|
var m bundle.Mutator
|
||||||
env := getEnvironment(cmd)
|
env := getTarget(cmd)
|
||||||
if env == "" {
|
if env == "" {
|
||||||
m = mutator.SelectDefaultEnvironment()
|
m = mutator.SelectDefaultTarget()
|
||||||
} else {
|
} else {
|
||||||
m = mutator.SelectEnvironment(env)
|
m = mutator.SelectTarget(env)
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx := cmd.Context()
|
ctx := cmd.Context()
|
||||||
|
@ -108,19 +124,27 @@ func TryConfigureBundle(cmd *cobra.Command, args []string) error {
|
||||||
return configureBundle(cmd, args, bundle.TryLoad)
|
return configureBundle(cmd, args, bundle.TryLoad)
|
||||||
}
|
}
|
||||||
|
|
||||||
// environmentCompletion executes to autocomplete the argument to the environment flag.
|
// targetCompletion executes to autocomplete the argument to the target flag.
|
||||||
func environmentCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
func targetCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
||||||
b, err := loadBundle(cmd, args, bundle.MustLoad)
|
b, err := loadBundle(cmd, args, bundle.MustLoad)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cobra.CompErrorln(err.Error())
|
cobra.CompErrorln(err.Error())
|
||||||
return nil, cobra.ShellCompDirectiveError
|
return nil, cobra.ShellCompDirectiveError
|
||||||
}
|
}
|
||||||
|
|
||||||
return maps.Keys(b.Config.Environments), cobra.ShellCompDirectiveDefault
|
return maps.Keys(b.Config.Targets), cobra.ShellCompDirectiveDefault
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func initTargetFlag(cmd *cobra.Command) {
|
||||||
|
// To operate in the context of a bundle, all commands must take an "target" parameter.
|
||||||
|
cmd.PersistentFlags().StringP("target", "t", "", "bundle target to use (if applicable)")
|
||||||
|
cmd.RegisterFlagCompletionFunc("target", targetCompletion)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DEPRECATED flag
|
||||||
func initEnvironmentFlag(cmd *cobra.Command) {
|
func initEnvironmentFlag(cmd *cobra.Command) {
|
||||||
// To operate in the context of a bundle, all commands must take an "environment" parameter.
|
// To operate in the context of a bundle, all commands must take an "environment" parameter.
|
||||||
cmd.PersistentFlags().StringP("environment", "e", "", "bundle environment to use (if applicable)")
|
cmd.PersistentFlags().StringP("environment", "e", "", "bundle target to use (if applicable)")
|
||||||
cmd.RegisterFlagCompletionFunc("environment", environmentCompletion)
|
cmd.PersistentFlags().MarkDeprecated("environment", "use --target flag instead")
|
||||||
|
cmd.RegisterFlagCompletionFunc("environment", targetCompletion)
|
||||||
}
|
}
|
||||||
|
|
|
@ -128,3 +128,27 @@ func TestBundleConfigureWithProfileFlagAndEnvVariable(t *testing.T) {
|
||||||
b.WorkspaceClient()
|
b.WorkspaceClient()
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTargetFlagFull(t *testing.T) {
|
||||||
|
cmd := emptyCommand(t)
|
||||||
|
initTargetFlag(cmd)
|
||||||
|
cmd.SetArgs([]string{"version", "--target", "development"})
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
err := cmd.ExecuteContext(ctx)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
assert.Equal(t, cmd.Flag("target").Value.String(), "development")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTargetFlagShort(t *testing.T) {
|
||||||
|
cmd := emptyCommand(t)
|
||||||
|
initTargetFlag(cmd)
|
||||||
|
cmd.SetArgs([]string{"version", "-t", "production"})
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
err := cmd.ExecuteContext(ctx)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
assert.Equal(t, cmd.Flag("target").Value.String(), "production")
|
||||||
|
}
|
||||||
|
|
|
@ -36,6 +36,7 @@ func New() *cobra.Command {
|
||||||
outputFlag := initOutputFlag(cmd)
|
outputFlag := initOutputFlag(cmd)
|
||||||
initProfileFlag(cmd)
|
initProfileFlag(cmd)
|
||||||
initEnvironmentFlag(cmd)
|
initEnvironmentFlag(cmd)
|
||||||
|
initTargetFlag(cmd)
|
||||||
|
|
||||||
cmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {
|
cmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {
|
||||||
ctx := cmd.Context()
|
ctx := cmd.Context()
|
||||||
|
|
|
@ -18,7 +18,7 @@ func TestSyncOptionsFromBundle(t *testing.T) {
|
||||||
Path: tempDir,
|
Path: tempDir,
|
||||||
|
|
||||||
Bundle: config.Bundle{
|
Bundle: config.Bundle{
|
||||||
Environment: "default",
|
Target: "default",
|
||||||
},
|
},
|
||||||
|
|
||||||
Workspace: config.Workspace{
|
Workspace: config.Workspace{
|
||||||
|
|
Loading…
Reference in New Issue