Add config environment support for variable overriding (#383)

## Changes
Allows to override default value for a variable definition from the
environment block in a bundle config. See bundle.yml for example usage

## Tests
Unit tests

---------

Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>

bundle/config/environment.go

@@ -14,4 +14,9 @@ type Environment struct {
 	Artifacts map[string]*Artifact `json:"artifacts,omitempty"`
 
 	Resources *Resources `json:"resources,omitempty"`
+
+	// Override default values for defined variables
+	// Does not permit defining new variables or redefining existing ones
+	// in the scope of an environment
+	Variables map[string]string `json:"variables,omitempty"`
 }

bundle/config/root.go

@@ -175,5 +175,17 @@ func (r *Root) MergeEnvironment(env *Environment) error {
 		}
 	}
 
+	if env.Variables != nil {
+		for k, v := range env.Variables {
+			variable, ok := r.Variables[k]
+			if !ok {
+				return fmt.Errorf("variable %s is not defined but is assigned a value", k)
+			}
+			// we only allow overrides of the default value for a variable
+			defaultVal := v
+			variable.Default = &defaultVal
+		}
+	}
+
 	return nil
 }

bundle/config/variable/variable.go

@@ -19,8 +19,9 @@ type Variable struct {
 	//
 	// 1. Command line flag. For example: `--var="foo=bar"`
 	// 2. Environment variable. eg: BUNDLE_VAR_foo=bar
-	// 3. default value defined in bundle config
+	// 3. Default value as defined in the applicable environments block
-	// 4. Throw error, since if no default value is defined, then the variable
+	// 4. Default value defined in variable definition
+	// 5. Throw error, since if no default value is defined, then the variable
 	//    is required
 	Value *string `json:"value,omitempty" bundle:"readonly"`
 }

bundle/tests/variables/env_overrides/bundle.yml

@@ -0,0 +1,32 @@
+variables:
+  a:
+    description: optional variable
+    default: default-a
+
+  b:
+    description: required variable
+
+bundle:
+  name: test bundle
+
+workspace:
+  profile: ${var.a} ${var.b}
+
+environments:
+  env-with-single-variable-override:
+    variables:
+      b: dev-b
+
+  env-missing-a-required-variable-assignment:
+    variables:
+      a: staging-a
+
+  env-with-two-variable-overrides:
+    variables:
+      a: prod-a
+      b: prod-b
+
+  env-using-an-undefined-variable:
+    variables:
+      c: prod-c
+      b: prod-b

bundle/tests/variables_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestVariables(t *testing.T) {
 	t.Setenv("BUNDLE_VAR_b", "def")
-	b := load(t, "./variables")
+	b := load(t, "./variables/vanilla")
 	err := bundle.Apply(context.Background(), b, []bundle.Mutator{
 		mutator.SetVariables(),
 		interpolation.Interpolate(
@@ -25,7 +25,7 @@ func TestVariables(t *testing.T) {
 }
 
 func TestVariablesLoadingFailsWhenRequiredVariableIsNotSpecified(t *testing.T) {
-	b := load(t, "./variables")
+	b := load(t, "./variables/vanilla")
 	err := bundle.Apply(context.Background(), b, []bundle.Mutator{
 		mutator.SetVariables(),
 		interpolation.Interpolate(
@@ -33,3 +33,62 @@ func TestVariablesLoadingFailsWhenRequiredVariableIsNotSpecified(t *testing.T) {
 		)})
 	assert.ErrorContains(t, err, "no value assigned to required variable b. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_b environment variable")
 }
+
+func TestVariablesEnvironmentsBlockOverride(t *testing.T) {
+	b := load(t, "./variables/env_overrides")
+	err := bundle.Apply(context.Background(), b, []bundle.Mutator{
+		mutator.SelectEnvironment("env-with-single-variable-override"),
+		mutator.SetVariables(),
+		interpolation.Interpolate(
+			interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
+		)})
+	require.NoError(t, err)
+	assert.Equal(t, "default-a dev-b", b.Config.Workspace.Profile)
+}
+
+func TestVariablesEnvironmentsBlockOverrideForMultipleVariables(t *testing.T) {
+	b := load(t, "./variables/env_overrides")
+	err := bundle.Apply(context.Background(), b, []bundle.Mutator{
+		mutator.SelectEnvironment("env-with-two-variable-overrides"),
+		mutator.SetVariables(),
+		interpolation.Interpolate(
+			interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
+		)})
+	require.NoError(t, err)
+	assert.Equal(t, "prod-a prod-b", b.Config.Workspace.Profile)
+}
+
+func TestVariablesEnvironmentsBlockOverrideWithProcessEnvVars(t *testing.T) {
+	t.Setenv("BUNDLE_VAR_b", "env-var-b")
+	b := load(t, "./variables/env_overrides")
+	err := bundle.Apply(context.Background(), b, []bundle.Mutator{
+		mutator.SelectEnvironment("env-with-two-variable-overrides"),
+		mutator.SetVariables(),
+		interpolation.Interpolate(
+			interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
+		)})
+	require.NoError(t, err)
+	assert.Equal(t, "prod-a env-var-b", b.Config.Workspace.Profile)
+}
+
+func TestVariablesEnvironmentsBlockOverrideWithMissingVariables(t *testing.T) {
+	b := load(t, "./variables/env_overrides")
+	err := bundle.Apply(context.Background(), b, []bundle.Mutator{
+		mutator.SelectEnvironment("env-missing-a-required-variable-assignment"),
+		mutator.SetVariables(),
+		interpolation.Interpolate(
+			interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
+		)})
+	assert.ErrorContains(t, err, "no value assigned to required variable b. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_b environment variable")
+}
+
+func TestVariablesEnvironmentsBlockOverrideWithUndefinedVariables(t *testing.T) {
+	b := load(t, "./variables/env_overrides")
+	err := bundle.Apply(context.Background(), b, []bundle.Mutator{
+		mutator.SelectEnvironment("env-using-an-undefined-variable"),
+		mutator.SetVariables(),
+		interpolation.Interpolate(
+			interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix),
+		)})
+	assert.ErrorContains(t, err, "variable c is not defined but is assigned a value")
+}