Reading variables from file (#2171)

## Changes

New source of default values for variables - variable file
`.databricks/bundle/<target>/variable-overrides.json`

CLI tries to stat and read that file every time during variable
initialisation phase

<!-- Summary of your changes that are easy to understand -->

## Tests

Acceptance tests

acceptance/bundle/variables/empty/output.txt

@@ -1,4 +1,4 @@
-Error: no value assigned to required variable a. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_a environment variable
+Error: no value assigned to required variable a. Assignment can be done using "--var", by setting the BUNDLE_VAR_a environment variable, or in .databricks/bundle/<target>/variable-overrides.json file
 
 Name: empty${var.a}
 Target: default

acceptance/bundle/variables/env_overrides/output.txt

@@ -9,7 +9,7 @@
 "prod-a env-var-b"
 
 >>> errcode $CLI bundle validate -t env-missing-a-required-variable-assignment
-Error: no value assigned to required variable b. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_b environment variable
+Error: no value assigned to required variable b. Assignment can be done using "--var", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle/<target>/variable-overrides.json file
 
 Name: test bundle
 Target: env-missing-a-required-variable-assignment

acceptance/bundle/variables/file-defaults/.databricks/bundle/complex_to_string/variable-overrides.json

@@ -0,0 +1,5 @@
+{
+  "cluster_key": {
+    "node_type_id": "Standard_DS3_v2"
+  }
+}

acceptance/bundle/variables/file-defaults/.databricks/bundle/default/variable-overrides.json

@@ -0,0 +1,7 @@
+{
+  "cluster": {
+    "node_type_id": "Standard_DS3_v2"
+  },
+  "cluster_key": "mlops_stacks-cluster",
+  "cluster_workers": 2
+}

acceptance/bundle/variables/file-defaults/.databricks/bundle/invalid_json/variable-overrides.json

@@ -0,0 +1 @@
+foo

acceptance/bundle/variables/file-defaults/.databricks/bundle/string_to_complex/variable-overrides.json

@@ -0,0 +1,3 @@
+{
+  "cluster": "mlops_stacks-cluster"
+}

acceptance/bundle/variables/file-defaults/.databricks/bundle/with_value/variable-overrides.json

@@ -0,0 +1,3 @@
+{
+  "cluster_key": "mlops_stacks-cluster-from-file"
+}

acceptance/bundle/variables/file-defaults/.databricks/bundle/without_defaults/variable-overrides.json

@@ -0,0 +1,4 @@
+{
+  "cluster_key": "mlops_stacks-cluster",
+  "cluster_workers": 2
+}

acceptance/bundle/variables/file-defaults/.databricks/bundle/wrong_file_structure/variable-overrides.json

@@ -0,0 +1,3 @@
+[
+  "foo"
+]

acceptance/bundle/variables/file-defaults/.gitignore

@@ -0,0 +1 @@
+!.databricks

acceptance/bundle/variables/file-defaults/databricks.yml

@@ -0,0 +1,53 @@
+bundle:
+  name: TestResolveVariablesFromFile
+
+variables:
+  cluster:
+    type: "complex"
+  cluster_key:
+  cluster_workers:
+
+resources:
+  jobs:
+    job1:
+      job_clusters:
+        - job_cluster_key: ${var.cluster_key}
+          new_cluster:
+            node_type_id: "${var.cluster.node_type_id}"
+            num_workers: ${var.cluster_workers}
+
+targets:
+  default:
+    default: true
+    variables:
+      cluster_workers: 1
+      cluster:
+        node_type_id: "default"
+      cluster_key: "default"
+
+  without_defaults:
+
+  complex_to_string:
+    variables:
+      cluster_workers: 1
+      cluster:
+        node_type_id: "default"
+      cluster_key: "default"
+
+  string_to_complex:
+    variables:
+      cluster_workers: 1
+      cluster:
+        node_type_id: "default"
+      cluster_key: "default"
+
+  wrong_file_structure:
+
+  invalid_json:
+
+  with_value:
+    variables:
+      cluster_workers: 1
+      cluster:
+        node_type_id: "default"
+      cluster_key: cluster_key_value

acceptance/bundle/variables/file-defaults/output.txt

@@ -0,0 +1,82 @@
+
+=== variable file
+>>> $CLI bundle validate -o json
+{
+  "job_cluster_key": "mlops_stacks-cluster",
+  "new_cluster": {
+    "node_type_id": "Standard_DS3_v2",
+    "num_workers": 2
+  }
+}
+
+=== variable file and variable flag
+>>> $CLI bundle validate -o json --var=cluster_key=mlops_stacks-cluster-overriden
+{
+  "job_cluster_key": "mlops_stacks-cluster-overriden",
+  "new_cluster": {
+    "node_type_id": "Standard_DS3_v2",
+    "num_workers": 2
+  }
+}
+
+=== variable file and environment variable
+>>> BUNDLE_VAR_cluster_key=mlops_stacks-cluster-overriden $CLI bundle validate -o json
+{
+  "job_cluster_key": "mlops_stacks-cluster-overriden",
+  "new_cluster": {
+    "node_type_id": "Standard_DS3_v2",
+    "num_workers": 2
+  }
+}
+
+=== variable has value in config file
+>>> $CLI bundle validate -o json --target with_value
+{
+  "job_cluster_key": "mlops_stacks-cluster-from-file",
+  "new_cluster": {
+    "node_type_id": "default",
+    "num_workers": 1
+  }
+}
+
+=== file has variable that is complex but default is string
+>>> errcode $CLI bundle validate -o json --target complex_to_string
+Error: variable cluster_key is not of type complex, but the value in the variable file is a complex type
+
+
+Exit code: 1
+{
+  "job_cluster_key": "${var.cluster_key}",
+  "new_cluster": {
+    "node_type_id": "${var.cluster.node_type_id}",
+    "num_workers": "${var.cluster_workers}"
+  }
+}
+
+=== file has variable that is string but default is complex
+>>> errcode $CLI bundle validate -o json --target string_to_complex
+Error: variable cluster is of type complex, but the value in the variable file is not a complex type
+
+
+Exit code: 1
+{
+  "job_cluster_key": "${var.cluster_key}",
+  "new_cluster": {
+    "node_type_id": "${var.cluster.node_type_id}",
+    "num_workers": "${var.cluster_workers}"
+  }
+}
+
+=== variable is required but it's not provided in the file
+>>> errcode $CLI bundle validate -o json --target without_defaults
+Error: no value assigned to required variable cluster. Assignment can be done using "--var", by setting the BUNDLE_VAR_cluster environment variable, or in .databricks/bundle/<target>/variable-overrides.json file
+
+
+Exit code: 1
+{
+  "job_cluster_key": "${var.cluster_key}",
+  "new_cluster": {
+    "node_type_id": "${var.cluster.node_type_id}",
+    "num_workers": "${var.cluster_workers}"
+  }
+}

acceptance/bundle/variables/file-defaults/script

@@ -0,0 +1,30 @@
+cluster_expr=".resources.jobs.job1.job_clusters[0]"
+
+# defaults from variable file, see .databricks/bundle/<target>/variable-overrides.json
+
+title "variable file"
+trace $CLI bundle validate -o json | jq $cluster_expr
+
+title "variable file and variable flag"
+trace $CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster-overriden" | jq $cluster_expr
+
+title "variable file and environment variable"
+trace BUNDLE_VAR_cluster_key=mlops_stacks-cluster-overriden $CLI bundle validate -o json | jq $cluster_expr
+
+title "variable has value in config file"
+trace $CLI bundle validate -o json --target with_value | jq $cluster_expr
+
+# title "file cannot be parsed"
+# trace errcode $CLI bundle validate -o json --target invalid_json | jq $cluster_expr
+
+# title "file has wrong structure"
+# trace errcode $CLI bundle validate -o json --target wrong_file_structure | jq $cluster_expr
+
+title "file has variable that is complex but default is string"
+trace errcode $CLI bundle validate -o json --target complex_to_string | jq $cluster_expr
+
+title "file has variable that is string but default is complex"
+trace errcode $CLI bundle validate -o json --target string_to_complex | jq $cluster_expr
+
+title "variable is required but it's not provided in the file"
+trace errcode $CLI bundle validate -o json --target without_defaults | jq $cluster_expr

acceptance/bundle/variables/vanilla/output.txt

@@ -3,7 +3,7 @@
 "abc def"
 
 >>> errcode $CLI bundle validate
-Error: no value assigned to required variable b. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_b environment variable
+Error: no value assigned to required variable b. Assignment can be done using "--var", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle/<target>/variable-overrides.json file
 
 Name: ${var.a} ${var.b}
 Target: default

acceptance/script.prepare

@@ -40,3 +40,8 @@ git-repo-init() {
     git add databricks.yml
     git commit -qm 'Add databricks.yml'
 }
+
+title() {
+    local label="$1"
+    printf "\n=== %s" "$label"
+}

bundle/config/mutator/set_variables.go

@@ -3,11 +3,14 @@ package mutator
 import (
 	"context"
 	"fmt"
+	"os"
+	"path/filepath"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config/variable"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/jsonloader"
 	"github.com/databricks/cli/libs/env"
 )
 
@@ -23,7 +26,11 @@ func (m *setVariables) Name() string {
 	return "SetVariables"
 }
 
-func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, name string) (dyn.Value, error) {
+func getDefaultVariableFilePath(target string) string {
+	return ".databricks/bundle/" + target + "/variable-overrides.json"
+}
+
+func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, name string, fileDefault dyn.Value) (dyn.Value, error) {
 	// case: variable already has value initialized, so skip
 	if variable.HasValue() {
 		return v, nil
@@ -49,6 +56,26 @@ func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable,
 		return v, nil
 	}
 
+	// case: Set the variable to the default value from the variable file
+	if fileDefault.Kind() != dyn.KindInvalid && fileDefault.Kind() != dyn.KindNil {
+		hasComplexType := variable.IsComplex()
+		hasComplexValue := fileDefault.Kind() == dyn.KindMap || fileDefault.Kind() == dyn.KindSequence
+
+		if hasComplexType && !hasComplexValue {
+			return dyn.InvalidValue, fmt.Errorf(`variable %s is of type complex, but the value in the variable file is not a complex type`, name)
+		}
+		if !hasComplexType && hasComplexValue {
+			return dyn.InvalidValue, fmt.Errorf(`variable %s is not of type complex, but the value in the variable file is a complex type`, name)
+		}
+
+		v, err := dyn.Set(v, "value", fileDefault)
+		if err != nil {
+			return dyn.InvalidValue, fmt.Errorf(`failed to assign default value from variable file to variable %s with error: %v`, name, err)
+		}
+
+		return v, nil
+	}
+
 	// case: Set the variable to its default value
 	if variable.HasDefault() {
 		vDefault, err := dyn.Get(v, "default")
@@ -64,10 +91,43 @@ func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable,
 	}
 
 	// We should have had a value to set for the variable at this point.
-	return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" flag or by setting the %s environment variable`, name, bundleVarPrefix+name)
+	return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done using "--var", by setting the %s environment variable, or in %s file`, name, bundleVarPrefix+name, getDefaultVariableFilePath("<target>"))
+}
+
+func readVariablesFromFile(b *bundle.Bundle) (dyn.Value, diag.Diagnostics) {
+	var diags diag.Diagnostics
+
+	filePath := filepath.Join(b.BundleRootPath, getDefaultVariableFilePath(b.Config.Bundle.Target))
+	if _, err := os.Stat(filePath); err != nil {
+		return dyn.InvalidValue, nil
+	}
+
+	f, err := os.ReadFile(filePath)
+	if err != nil {
+		return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to read variables file: %w", err))
+	}
+
+	val, err := jsonloader.LoadJSON(f, filePath)
+	if err != nil {
+		return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to parse variables file %s: %w", filePath, err))
+	}
+
+	if val.Kind() != dyn.KindMap {
+		return dyn.InvalidValue, diags.Append(diag.Diagnostic{
+			Severity: diag.Error,
+			Summary:  fmt.Sprintf("failed to parse variables file %s: invalid format", filePath),
+			Detail:   "Variables file must be a JSON object with the following format:\n{\"var1\": \"value1\", \"var2\": \"value2\"}",
+		})
+	}
+
+	return val, nil
 }
 
 func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	defaults, diags := readVariablesFromFile(b)
+	if diags.HasError() {
+		return diags
+	}
 	err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
 		return dyn.Map(v, "variables", dyn.Foreach(func(p dyn.Path, variable dyn.Value) (dyn.Value, error) {
 			name := p[1].Key()
@@ -76,9 +136,10 @@ func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 				return dyn.InvalidValue, fmt.Errorf(`variable "%s" is not defined`, name)
 			}
 
-			return setVariable(ctx, variable, v, name)
+			fileDefault, _ := dyn.Get(defaults, name)
+			return setVariable(ctx, variable, v, name, fileDefault)
 		}))
 	})
 
-	return diag.FromErr(err)
+	return diags.Extend(diag.FromErr(err))
 }

bundle/config/mutator/set_variables_test.go

@@ -25,7 +25,7 @@ func TestSetVariableFromProcessEnvVar(t *testing.T) {
 	v, err := convert.FromTyped(variable, dyn.NilValue)
 	require.NoError(t, err)
 
-	v, err = setVariable(context.Background(), v, &variable, "foo")
+	v, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue)
 	require.NoError(t, err)
 
 	err = convert.ToTyped(&variable, v)
@@ -43,7 +43,7 @@ func TestSetVariableUsingDefaultValue(t *testing.T) {
 	v, err := convert.FromTyped(variable, dyn.NilValue)
 	require.NoError(t, err)
 
-	v, err = setVariable(context.Background(), v, &variable, "foo")
+	v, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue)
 	require.NoError(t, err)
 
 	err = convert.ToTyped(&variable, v)
@@ -65,7 +65,7 @@ func TestSetVariableWhenAlreadyAValueIsAssigned(t *testing.T) {
 	v, err := convert.FromTyped(variable, dyn.NilValue)
 	require.NoError(t, err)
 
-	v, err = setVariable(context.Background(), v, &variable, "foo")
+	v, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue)
 	require.NoError(t, err)
 
 	err = convert.ToTyped(&variable, v)
@@ -90,7 +90,7 @@ func TestSetVariableEnvVarValueDoesNotOverridePresetValue(t *testing.T) {
 	v, err := convert.FromTyped(variable, dyn.NilValue)
 	require.NoError(t, err)
 
-	v, err = setVariable(context.Background(), v, &variable, "foo")
+	v, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue)
 	require.NoError(t, err)
 
 	err = convert.ToTyped(&variable, v)
@@ -107,8 +107,8 @@ func TestSetVariablesErrorsIfAValueCouldNotBeResolved(t *testing.T) {
 	v, err := convert.FromTyped(variable, dyn.NilValue)
 	require.NoError(t, err)
 
-	_, err = setVariable(context.Background(), v, &variable, "foo")
-	assert.ErrorContains(t, err, "no value assigned to required variable foo. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_foo environment variable")
+	_, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue)
+	assert.ErrorContains(t, err, "no value assigned to required variable foo. Assignment can be done using \"--var\", by setting the BUNDLE_VAR_foo environment variable, or in .databricks/bundle/<target>/variable-overrides.json file")
 }
 
 func TestSetVariablesMutator(t *testing.T) {
@@ -157,6 +157,6 @@ func TestSetComplexVariablesViaEnvVariablesIsNotAllowed(t *testing.T) {
 	v, err := convert.FromTyped(variable, dyn.NilValue)
 	require.NoError(t, err)
 
-	_, err = setVariable(context.Background(), v, &variable, "foo")
+	_, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue)
 	assert.ErrorContains(t, err, "setting via environment variables (BUNDLE_VAR_foo) is not supported for complex variable foo")
 }

bundle/config/variable/variable.go

@@ -36,11 +36,12 @@ type Variable struct {
 	// This field stores the resolved value for the variable. The variable are
 	// resolved in the following priority order (from highest to lowest)
 	//
-	// 1. Command line flag. For example: `--var="foo=bar"`
-	// 2. Target variable. eg: BUNDLE_VAR_foo=bar
-	// 3. Default value as defined in the applicable environments block
-	// 4. Default value defined in variable definition
-	// 5. Throw error, since if no default value is defined, then the variable
+	// 1. Command line flag `--var="foo=bar"`
+	// 2. Environment variable. eg: BUNDLE_VAR_foo=bar
+	// 3. Load defaults from .databricks/bundle/<target>/variable-overrides.json
+	// 4. Default value as defined in the applicable targets block
+	// 5. Default value defined in variable definition
+	// 6. Throw error, since if no default value is defined, then the variable
 	//    is required
 	Value VariableValue `json:"value,omitempty" bundle:"readonly"`