feat: Default value for variable path

acceptance/bundle/variables/var_file_overrides/.databricks/bundle/with-default-variable-file/vars.json

@@ -0,0 +1,7 @@
+{
+  "cluster": {
+    "node_type_id": "Standard_DS3_v3"
+  },
+  "cluster_key": "mlops_stacks-cluster-2",
+  "cluster_workers": 9
+}

acceptance/bundle/variables/var_file_overrides/.gitignore

@@ -0,0 +1 @@
+!.databricks

acceptance/bundle/variables/var_file_overrides/databricks.yml

@@ -30,3 +30,6 @@ targets:
         default: "default"
 
   without-defaults:
+
+  # see .databricks/bundle/default_target/ for variable values
+  with-default-variable-file:

acceptance/bundle/variables/var_file_overrides/output.txt

@@ -8,25 +8,30 @@
   }
 }
 
->>> $CLI bundle validate -o json --var=cluster_key=mlops_stacks-cluster
+>>> $CLI bundle validate -o json --target with-default-variable-file
 {
-  "job_cluster_key": "mlops_stacks-cluster",
+  "job_cluster_key": "mlops_stacks-cluster-2",
   "new_cluster": {
-    "node_type_id": "default",
+    "node_type_id": "Standard_DS3_v3",
-    "num_workers": 1
+    "num_workers": 9
   }
 }
 
->>> errcode $CLI bundle validate -o json --var-file=var_files/normal.json --var=cluster_key=mlops_stacks-cluster
+>>> $CLI bundle validate -o json --var-file=var_files/normal.json --var=cluster_key=mlops_stacks-cluster-overriden
-Error: cannot specify both --var and --var-file flags
-
-
-Exit code: 1
 {
-  "job_cluster_key": "${var.cluster_key}",
+  "job_cluster_key": "mlops_stacks-cluster-overriden",
   "new_cluster": {
-    "node_type_id": "${var.cluster.node_type_id}",
+    "node_type_id": "Standard_DS3_v2",
-    "num_workers": "${var.cluster_workers}"
+    "num_workers": 2
+  }
+}
+
+>>> BUNDLE_VAR_cluster_key=incorrectly-overriden $CLI bundle validate -o json --var-file=var_files/normal.json
+{
+  "job_cluster_key": "mlops_stacks-cluster",
+  "new_cluster": {
+    "node_type_id": "Standard_DS3_v2",
+    "num_workers": 2
   }
 }
 
@@ -113,7 +118,7 @@ Exit code: 1
 }
 
 >>> errcode $CLI bundle validate -o json --target without-defaults --var-file=var_files/without_required.json
-Error: no value assigned to required variable cluster. Assignment can be done through the "--var" or "--var-file" flag or by setting the BUNDLE_VAR_cluster environment variable
+Error: no value assigned to required variable cluster. Assignment can be done using "--var" or "--var-file", by setting the BUNDLE_VAR_cluster environment variable, or in .databricks/bundle/<target>/vars.json file
 
 
 Exit code: 1

acceptance/bundle/variables/var_file_overrides/script

@@ -3,11 +3,14 @@ cluster_expr=".resources.jobs.job1.job_clusters[0]"
 # variable file
 trace $CLI bundle validate -o json --var-file=var_files/normal.json | jq $cluster_expr
 
-# variable flag
+# default variable file (see .databricks/*)
-trace $CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster" | jq $cluster_expr
+trace $CLI bundle validate -o json --target with-default-variable-file | jq $cluster_expr
 
-# both variable file and flag
+# variable file and variable flag
-trace errcode $CLI bundle validate -o json --var-file=var_files/normal.json --var="cluster_key=mlops_stacks-cluster" | jq $cluster_expr
+trace $CLI bundle validate -o json --var-file=var_files/normal.json --var="cluster_key=mlops_stacks-cluster-overriden" | jq $cluster_expr
+
+# variable file and environment variable
+trace BUNDLE_VAR_cluster_key=incorrectly-overriden $CLI bundle validate -o json --var-file=var_files/normal.json | jq $cluster_expr
 
 # file not found
 trace errcode $CLI bundle validate -o json --var-file=var_files/not_found.json 2> >(sed 's/\(Error: failed to read variables file: open var_files\/not_found.json:\).*/\1<stripped>/' >&2) > tmp.txt

bundle/config/mutator/set_variables.go

@@ -64,7 +64,7 @@ func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable,
 	}
 
 	// We should have had a value to set for the variable at this point.
-	return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" or "--var-file" flag or by setting the %s environment variable`, name, bundleVarPrefix+name)
+	return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done using "--var" or "--var-file", by setting the %s environment variable, or in .databricks/bundle/<target>/vars.json file`, name, bundleVarPrefix+name)
 }
 
 func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {

bundle/config/root.go

@@ -269,6 +269,9 @@ func (r *Root) InitializeAnyTypeVariables(vars map[string]any) error {
 			return fmt.Errorf("variable %s has not been defined", name)
 		}
 
+		if variable.HasValue() {
+			continue
+		}
 		err := variable.Set(val)
 		if err != nil {
 			return fmt.Errorf("failed to assign %s to %s: %s", val, name, err)

bundle/config/variable/variable.go

@@ -36,13 +36,13 @@ type Variable struct {
 	// This field stores the resolved value for the variable. The variable are
 	// resolved in the following priority order (from highest to lowest)
 	//
-	// 1. Command line flag, one of these is used
+	// 1. Command line flag `--var="foo=bar"`
-	//   a. Variable value obtained from arguments, example: `--var="foo=bar"`
+	// 2. Variable value from the file, example: `--var-file="/path/to/file"`.
-	//   b. Variable value obtained from the file, example: `--var-file="/path/to/file"`
+	//    If path is not specified the default path is used: ".databricks/bundle/<target>/vars.json"
-	// 2. Environment variable. eg: BUNDLE_VAR_foo=bar
+	// 3. Environment variable. eg: BUNDLE_VAR_foo=bar
-	// 3. Default value as defined in the applicable targets block
+	// 4. Default value as defined in the applicable targets block
-	// 4. Default value defined in variable definition
+	// 5. Default value defined in variable definition
-	// 5. Throw error, since if no default value is defined, then the variable
+	// 6. Throw error, since if no default value is defined, then the variable
 	//    is required
 	Value VariableValue `json:"value,omitempty" bundle:"readonly"`
 

cmd/bundle/utils/utils.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path/filepath"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/cmd/root"
@@ -13,6 +14,10 @@ import (
 	"github.com/spf13/cobra"
 )
 
+func GetDefaultVariableFilePath(target string) string {
+	return ".databricks/bundle/" + target + "/vars.json"
+}
+
 func configureVariables(cmd *cobra.Command, b *bundle.Bundle, variables []string) diag.Diagnostics {
 	return bundle.ApplyFunc(cmd.Context(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 		err := b.Config.InitializeVariables(variables)
@@ -66,17 +71,27 @@ func ConfigureBundleWithVariables(cmd *cobra.Command) (*bundle.Bundle, diag.Diag
 	if err != nil {
 		return b, diag.FromErr(err)
 	}
+
+	if len(variables) > 0 {
+		// Initialize variables by assigning them values passed as command line flags
+		diags = diags.Extend(configureVariables(cmd, b, variables))
+	}
+
 	variableFilePath, err := cmd.Flags().GetString("var-file")
 	if err != nil {
 		return b, diag.FromErr(err)
 	}
 
-	if len(variables) > 0 && variableFilePath != "" {
+	if variableFilePath == "" {
-		return b, diag.Errorf("cannot specify both --var and --var-file flags")
+		// Fallback to default variable file path
-	} else if len(variables) > 0 {
+		defaultPath := GetDefaultVariableFilePath(b.Config.Bundle.Target)
-		// Initialize variables by assigning them values passed as command line flags
+		normalisedPath := filepath.Join(b.BundleRootPath, defaultPath)
-		diags = diags.Extend(configureVariables(cmd, b, variables))
+		if _, err := os.Stat(normalisedPath); err == nil {
-	} else if variableFilePath != "" {
+			variableFilePath = normalisedPath
+		}
+	}
+
+	if variableFilePath != "" {
 		// Initialize variables by loading them from a file
 		diags = diags.Extend(configureVariablesFromFile(cmd, b, variableFilePath))
 	}

cmd/bundle/variables.go

@@ -1,10 +1,14 @@
 package bundle
 
 import (
+	"fmt"
+
 	"github.com/spf13/cobra"
+
+	"github.com/databricks/cli/cmd/bundle/utils"
 )
 
 func initVariableFlag(cmd *cobra.Command) {
 	cmd.PersistentFlags().StringSlice("var", []string{}, `set values for variables defined in bundle config. Example: --var="foo=bar"`)
-	cmd.PersistentFlags().String("var-file", "", `file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json"`)
+	cmd.PersistentFlags().String("var-file", "", fmt.Sprintf(`file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default "%s")`, utils.GetDefaultVariableFilePath("<target>")))
 }