diff --git a/acceptance/bundle/variables/var_file_overrides/.databricks/bundle/with-default-variable-file/vars.json b/acceptance/bundle/variables/var_file_overrides/.databricks/bundle/with-default-variable-file/vars.json new file mode 100644 index 000000000..035ff1339 --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/.databricks/bundle/with-default-variable-file/vars.json @@ -0,0 +1,7 @@ +{ + "cluster": { + "node_type_id": "Standard_DS3_v3" + }, + "cluster_key": "mlops_stacks-cluster-2", + "cluster_workers": 9 +} diff --git a/acceptance/bundle/variables/var_file_overrides/.gitignore b/acceptance/bundle/variables/var_file_overrides/.gitignore new file mode 100644 index 000000000..bd1711fd1 --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/.gitignore @@ -0,0 +1 @@ +!.databricks diff --git a/acceptance/bundle/variables/var_file_overrides/databricks.yml b/acceptance/bundle/variables/var_file_overrides/databricks.yml index cf743a556..3c8de8551 100644 --- a/acceptance/bundle/variables/var_file_overrides/databricks.yml +++ b/acceptance/bundle/variables/var_file_overrides/databricks.yml @@ -30,3 +30,6 @@ targets: default: "default" without-defaults: + + # see .databricks/bundle/default_target/ for variable values + with-default-variable-file: diff --git a/acceptance/bundle/variables/var_file_overrides/output.txt b/acceptance/bundle/variables/var_file_overrides/output.txt index c2c73dff9..f5b9d20b1 100644 --- a/acceptance/bundle/variables/var_file_overrides/output.txt +++ b/acceptance/bundle/variables/var_file_overrides/output.txt @@ -8,25 +8,30 @@ } } ->>> $CLI bundle validate -o json --var=cluster_key=mlops_stacks-cluster +>>> $CLI bundle validate -o json --target with-default-variable-file { - "job_cluster_key": "mlops_stacks-cluster", + "job_cluster_key": "mlops_stacks-cluster-2", "new_cluster": { - "node_type_id": "default", - "num_workers": 1 + "node_type_id": "Standard_DS3_v3", + "num_workers": 9 } } ->>> errcode $CLI bundle validate -o json --var-file=var_files/normal.json --var=cluster_key=mlops_stacks-cluster -Error: cannot specify both --var and --var-file flags - - -Exit code: 1 +>>> $CLI bundle validate -o json --var-file=var_files/normal.json --var=cluster_key=mlops_stacks-cluster-overriden { - "job_cluster_key": "${var.cluster_key}", + "job_cluster_key": "mlops_stacks-cluster-overriden", "new_cluster": { - "node_type_id": "${var.cluster.node_type_id}", - "num_workers": "${var.cluster_workers}" + "node_type_id": "Standard_DS3_v2", + "num_workers": 2 + } +} + +>>> BUNDLE_VAR_cluster_key=incorrectly-overriden $CLI bundle validate -o json --var-file=var_files/normal.json +{ + "job_cluster_key": "mlops_stacks-cluster", + "new_cluster": { + "node_type_id": "Standard_DS3_v2", + "num_workers": 2 } } @@ -113,7 +118,7 @@ Exit code: 1 } >>> errcode $CLI bundle validate -o json --target without-defaults --var-file=var_files/without_required.json -Error: no value assigned to required variable cluster. Assignment can be done through the "--var" or "--var-file" flag or by setting the BUNDLE_VAR_cluster environment variable +Error: no value assigned to required variable cluster. Assignment can be done using "--var" or "--var-file", by setting the BUNDLE_VAR_cluster environment variable, or in .databricks/bundle//vars.json file Exit code: 1 diff --git a/acceptance/bundle/variables/var_file_overrides/script b/acceptance/bundle/variables/var_file_overrides/script index b073409ec..9dab54bda 100644 --- a/acceptance/bundle/variables/var_file_overrides/script +++ b/acceptance/bundle/variables/var_file_overrides/script @@ -3,11 +3,14 @@ cluster_expr=".resources.jobs.job1.job_clusters[0]" # variable file trace $CLI bundle validate -o json --var-file=var_files/normal.json | jq $cluster_expr -# variable flag -trace $CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster" | jq $cluster_expr +# default variable file (see .databricks/*) +trace $CLI bundle validate -o json --target with-default-variable-file | jq $cluster_expr -# both variable file and flag -trace errcode $CLI bundle validate -o json --var-file=var_files/normal.json --var="cluster_key=mlops_stacks-cluster" | jq $cluster_expr +# variable file and variable flag +trace $CLI bundle validate -o json --var-file=var_files/normal.json --var="cluster_key=mlops_stacks-cluster-overriden" | jq $cluster_expr + +# variable file and environment variable +trace BUNDLE_VAR_cluster_key=incorrectly-overriden $CLI bundle validate -o json --var-file=var_files/normal.json | jq $cluster_expr # file not found trace errcode $CLI bundle validate -o json --var-file=var_files/not_found.json 2> >(sed 's/\(Error: failed to read variables file: open var_files\/not_found.json:\).*/\1/' >&2) > tmp.txt diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go index 7b16571e2..d94f52547 100644 --- a/bundle/config/mutator/set_variables.go +++ b/bundle/config/mutator/set_variables.go @@ -64,7 +64,7 @@ func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, } // We should have had a value to set for the variable at this point. - return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" or "--var-file" flag or by setting the %s environment variable`, name, bundleVarPrefix+name) + return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done using "--var" or "--var-file", by setting the %s environment variable, or in .databricks/bundle//vars.json file`, name, bundleVarPrefix+name) } func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diff --git a/bundle/config/root.go b/bundle/config/root.go index 3b3c6e5ce..0bcf9d3b9 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -269,6 +269,9 @@ func (r *Root) InitializeAnyTypeVariables(vars map[string]any) error { return fmt.Errorf("variable %s has not been defined", name) } + if variable.HasValue() { + continue + } err := variable.Set(val) if err != nil { return fmt.Errorf("failed to assign %s to %s: %s", val, name, err) diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go index 76feebaad..fd2556ed6 100644 --- a/bundle/config/variable/variable.go +++ b/bundle/config/variable/variable.go @@ -36,13 +36,13 @@ type Variable struct { // This field stores the resolved value for the variable. The variable are // resolved in the following priority order (from highest to lowest) // - // 1. Command line flag, one of these is used - // a. Variable value obtained from arguments, example: `--var="foo=bar"` - // b. Variable value obtained from the file, example: `--var-file="/path/to/file"` - // 2. Environment variable. eg: BUNDLE_VAR_foo=bar - // 3. Default value as defined in the applicable targets block - // 4. Default value defined in variable definition - // 5. Throw error, since if no default value is defined, then the variable + // 1. Command line flag `--var="foo=bar"` + // 2. Variable value from the file, example: `--var-file="/path/to/file"`. + // If path is not specified the default path is used: ".databricks/bundle//vars.json" + // 3. Environment variable. eg: BUNDLE_VAR_foo=bar + // 4. Default value as defined in the applicable targets block + // 5. Default value defined in variable definition + // 6. Throw error, since if no default value is defined, then the variable // is required Value VariableValue `json:"value,omitempty" bundle:"readonly"` diff --git a/cmd/bundle/utils/utils.go b/cmd/bundle/utils/utils.go index f58ef274c..e659ae600 100644 --- a/cmd/bundle/utils/utils.go +++ b/cmd/bundle/utils/utils.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "path/filepath" "github.com/databricks/cli/bundle" "github.com/databricks/cli/cmd/root" @@ -13,6 +14,10 @@ import ( "github.com/spf13/cobra" ) +func GetDefaultVariableFilePath(target string) string { + return ".databricks/bundle/" + target + "/vars.json" +} + func configureVariables(cmd *cobra.Command, b *bundle.Bundle, variables []string) diag.Diagnostics { return bundle.ApplyFunc(cmd.Context(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { err := b.Config.InitializeVariables(variables) @@ -66,17 +71,27 @@ func ConfigureBundleWithVariables(cmd *cobra.Command) (*bundle.Bundle, diag.Diag if err != nil { return b, diag.FromErr(err) } + + if len(variables) > 0 { + // Initialize variables by assigning them values passed as command line flags + diags = diags.Extend(configureVariables(cmd, b, variables)) + } + variableFilePath, err := cmd.Flags().GetString("var-file") if err != nil { return b, diag.FromErr(err) } - if len(variables) > 0 && variableFilePath != "" { - return b, diag.Errorf("cannot specify both --var and --var-file flags") - } else if len(variables) > 0 { - // Initialize variables by assigning them values passed as command line flags - diags = diags.Extend(configureVariables(cmd, b, variables)) - } else if variableFilePath != "" { + if variableFilePath == "" { + // Fallback to default variable file path + defaultPath := GetDefaultVariableFilePath(b.Config.Bundle.Target) + normalisedPath := filepath.Join(b.BundleRootPath, defaultPath) + if _, err := os.Stat(normalisedPath); err == nil { + variableFilePath = normalisedPath + } + } + + if variableFilePath != "" { // Initialize variables by loading them from a file diags = diags.Extend(configureVariablesFromFile(cmd, b, variableFilePath)) } diff --git a/cmd/bundle/variables.go b/cmd/bundle/variables.go index e409e8946..984d214e4 100644 --- a/cmd/bundle/variables.go +++ b/cmd/bundle/variables.go @@ -1,10 +1,14 @@ package bundle import ( + "fmt" + "github.com/spf13/cobra" + + "github.com/databricks/cli/cmd/bundle/utils" ) func initVariableFlag(cmd *cobra.Command) { cmd.PersistentFlags().StringSlice("var", []string{}, `set values for variables defined in bundle config. Example: --var="foo=bar"`) - cmd.PersistentFlags().String("var-file", "", `file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json"`) + cmd.PersistentFlags().String("var-file", "", fmt.Sprintf(`file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default "%s")`, utils.GetDefaultVariableFilePath(""))) }