diff --git a/acceptance/bundle/variables/empty/output.txt b/acceptance/bundle/variables/empty/output.txt index 261635920..8933443df 100644 --- a/acceptance/bundle/variables/empty/output.txt +++ b/acceptance/bundle/variables/empty/output.txt @@ -1,4 +1,4 @@ -Error: no value assigned to required variable a. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_a environment variable +Error: no value assigned to required variable a. Assignment can be done using "--var", by setting the BUNDLE_VAR_a environment variable, or in .databricks/bundle//variable-overrides.json file Name: empty${var.a} Target: default diff --git a/acceptance/bundle/variables/env_overrides/output.txt b/acceptance/bundle/variables/env_overrides/output.txt index f42f82211..1ee9ef625 100644 --- a/acceptance/bundle/variables/env_overrides/output.txt +++ b/acceptance/bundle/variables/env_overrides/output.txt @@ -9,7 +9,7 @@ "prod-a env-var-b" >>> errcode $CLI bundle validate -t env-missing-a-required-variable-assignment -Error: no value assigned to required variable b. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_b environment variable +Error: no value assigned to required variable b. Assignment can be done using "--var", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle//variable-overrides.json file Name: test bundle Target: env-missing-a-required-variable-assignment diff --git a/acceptance/bundle/variables/file-defaults/.databricks/bundle/complex_to_string/variable-overrides.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/complex_to_string/variable-overrides.json new file mode 100644 index 000000000..602567a68 --- /dev/null +++ b/acceptance/bundle/variables/file-defaults/.databricks/bundle/complex_to_string/variable-overrides.json @@ -0,0 +1,5 @@ +{ + "cluster_key": { + "node_type_id": "Standard_DS3_v2" + } +} diff --git a/acceptance/bundle/variables/file-defaults/.databricks/bundle/default/variable-overrides.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/default/variable-overrides.json new file mode 100644 index 000000000..3a865e120 --- /dev/null +++ b/acceptance/bundle/variables/file-defaults/.databricks/bundle/default/variable-overrides.json @@ -0,0 +1,7 @@ +{ + "cluster": { + "node_type_id": "Standard_DS3_v2" + }, + "cluster_key": "mlops_stacks-cluster", + "cluster_workers": 2 +} diff --git a/acceptance/bundle/variables/file-defaults/.databricks/bundle/invalid_json/variable-overrides.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/invalid_json/variable-overrides.json new file mode 100644 index 000000000..257cc5642 --- /dev/null +++ b/acceptance/bundle/variables/file-defaults/.databricks/bundle/invalid_json/variable-overrides.json @@ -0,0 +1 @@ +foo diff --git a/acceptance/bundle/variables/file-defaults/.databricks/bundle/string_to_complex/variable-overrides.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/string_to_complex/variable-overrides.json new file mode 100644 index 000000000..1ea719446 --- /dev/null +++ b/acceptance/bundle/variables/file-defaults/.databricks/bundle/string_to_complex/variable-overrides.json @@ -0,0 +1,3 @@ +{ + "cluster": "mlops_stacks-cluster" +} diff --git a/acceptance/bundle/variables/file-defaults/.databricks/bundle/with_value/variable-overrides.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/with_value/variable-overrides.json new file mode 100644 index 000000000..686d68548 --- /dev/null +++ b/acceptance/bundle/variables/file-defaults/.databricks/bundle/with_value/variable-overrides.json @@ -0,0 +1,3 @@ +{ + "cluster_key": "mlops_stacks-cluster-from-file" +} diff --git a/acceptance/bundle/variables/file-defaults/.databricks/bundle/without_defaults/variable-overrides.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/without_defaults/variable-overrides.json new file mode 100644 index 000000000..86166408e --- /dev/null +++ b/acceptance/bundle/variables/file-defaults/.databricks/bundle/without_defaults/variable-overrides.json @@ -0,0 +1,4 @@ +{ + "cluster_key": "mlops_stacks-cluster", + "cluster_workers": 2 +} diff --git a/acceptance/bundle/variables/file-defaults/.databricks/bundle/wrong_file_structure/variable-overrides.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/wrong_file_structure/variable-overrides.json new file mode 100644 index 000000000..de140ba36 --- /dev/null +++ b/acceptance/bundle/variables/file-defaults/.databricks/bundle/wrong_file_structure/variable-overrides.json @@ -0,0 +1,3 @@ +[ + "foo" +] diff --git a/acceptance/bundle/variables/file-defaults/.gitignore b/acceptance/bundle/variables/file-defaults/.gitignore new file mode 100644 index 000000000..bd1711fd1 --- /dev/null +++ b/acceptance/bundle/variables/file-defaults/.gitignore @@ -0,0 +1 @@ +!.databricks diff --git a/acceptance/bundle/variables/file-defaults/databricks.yml b/acceptance/bundle/variables/file-defaults/databricks.yml new file mode 100644 index 000000000..5838843e1 --- /dev/null +++ b/acceptance/bundle/variables/file-defaults/databricks.yml @@ -0,0 +1,53 @@ +bundle: + name: TestResolveVariablesFromFile + +variables: + cluster: + type: "complex" + cluster_key: + cluster_workers: + +resources: + jobs: + job1: + job_clusters: + - job_cluster_key: ${var.cluster_key} + new_cluster: + node_type_id: "${var.cluster.node_type_id}" + num_workers: ${var.cluster_workers} + +targets: + default: + default: true + variables: + cluster_workers: 1 + cluster: + node_type_id: "default" + cluster_key: "default" + + without_defaults: + + complex_to_string: + variables: + cluster_workers: 1 + cluster: + node_type_id: "default" + cluster_key: "default" + + string_to_complex: + variables: + cluster_workers: 1 + cluster: + node_type_id: "default" + cluster_key: "default" + + wrong_file_structure: + + invalid_json: + + with_value: + variables: + cluster_workers: 1 + cluster: + node_type_id: "default" + cluster_key: cluster_key_value diff --git a/acceptance/bundle/variables/file-defaults/output.txt b/acceptance/bundle/variables/file-defaults/output.txt new file mode 100644 index 000000000..73830aae3 --- /dev/null +++ b/acceptance/bundle/variables/file-defaults/output.txt @@ -0,0 +1,82 @@ + +=== variable file +>>> $CLI bundle validate -o json +{ + "job_cluster_key": "mlops_stacks-cluster", + "new_cluster": { + "node_type_id": "Standard_DS3_v2", + "num_workers": 2 + } +} + +=== variable file and variable flag +>>> $CLI bundle validate -o json --var=cluster_key=mlops_stacks-cluster-overriden +{ + "job_cluster_key": "mlops_stacks-cluster-overriden", + "new_cluster": { + "node_type_id": "Standard_DS3_v2", + "num_workers": 2 + } +} + +=== variable file and environment variable +>>> BUNDLE_VAR_cluster_key=mlops_stacks-cluster-overriden $CLI bundle validate -o json +{ + "job_cluster_key": "mlops_stacks-cluster-overriden", + "new_cluster": { + "node_type_id": "Standard_DS3_v2", + "num_workers": 2 + } +} + +=== variable has value in config file +>>> $CLI bundle validate -o json --target with_value +{ + "job_cluster_key": "mlops_stacks-cluster-from-file", + "new_cluster": { + "node_type_id": "default", + "num_workers": 1 + } +} + +=== file has variable that is complex but default is string +>>> errcode $CLI bundle validate -o json --target complex_to_string +Error: variable cluster_key is not of type complex, but the value in the variable file is a complex type + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + +=== file has variable that is string but default is complex +>>> errcode $CLI bundle validate -o json --target string_to_complex +Error: variable cluster is of type complex, but the value in the variable file is not a complex type + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + +=== variable is required but it's not provided in the file +>>> errcode $CLI bundle validate -o json --target without_defaults +Error: no value assigned to required variable cluster. Assignment can be done using "--var", by setting the BUNDLE_VAR_cluster environment variable, or in .databricks/bundle//variable-overrides.json file + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} diff --git a/acceptance/bundle/variables/file-defaults/script b/acceptance/bundle/variables/file-defaults/script new file mode 100644 index 000000000..c5b208755 --- /dev/null +++ b/acceptance/bundle/variables/file-defaults/script @@ -0,0 +1,30 @@ +cluster_expr=".resources.jobs.job1.job_clusters[0]" + +# defaults from variable file, see .databricks/bundle//variable-overrides.json + +title "variable file" +trace $CLI bundle validate -o json | jq $cluster_expr + +title "variable file and variable flag" +trace $CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster-overriden" | jq $cluster_expr + +title "variable file and environment variable" +trace BUNDLE_VAR_cluster_key=mlops_stacks-cluster-overriden $CLI bundle validate -o json | jq $cluster_expr + +title "variable has value in config file" +trace $CLI bundle validate -o json --target with_value | jq $cluster_expr + +# title "file cannot be parsed" +# trace errcode $CLI bundle validate -o json --target invalid_json | jq $cluster_expr + +# title "file has wrong structure" +# trace errcode $CLI bundle validate -o json --target wrong_file_structure | jq $cluster_expr + +title "file has variable that is complex but default is string" +trace errcode $CLI bundle validate -o json --target complex_to_string | jq $cluster_expr + +title "file has variable that is string but default is complex" +trace errcode $CLI bundle validate -o json --target string_to_complex | jq $cluster_expr + +title "variable is required but it's not provided in the file" +trace errcode $CLI bundle validate -o json --target without_defaults | jq $cluster_expr diff --git a/acceptance/bundle/variables/vanilla/output.txt b/acceptance/bundle/variables/vanilla/output.txt index 1d88bd060..e98882bb0 100644 --- a/acceptance/bundle/variables/vanilla/output.txt +++ b/acceptance/bundle/variables/vanilla/output.txt @@ -3,7 +3,7 @@ "abc def" >>> errcode $CLI bundle validate -Error: no value assigned to required variable b. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_b environment variable +Error: no value assigned to required variable b. Assignment can be done using "--var", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle//variable-overrides.json file Name: ${var.a} ${var.b} Target: default diff --git a/acceptance/script.prepare b/acceptance/script.prepare index 5900016d7..0567e433a 100644 --- a/acceptance/script.prepare +++ b/acceptance/script.prepare @@ -40,3 +40,8 @@ git-repo-init() { git add databricks.yml git commit -qm 'Add databricks.yml' } + +title() { + local label="$1" + printf "\n=== %s" "$label" +} diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go index 9e9f2dcfe..ac2f660a9 100644 --- a/bundle/config/mutator/set_variables.go +++ b/bundle/config/mutator/set_variables.go @@ -3,11 +3,14 @@ package mutator import ( "context" "fmt" + "os" + "path/filepath" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/jsonloader" "github.com/databricks/cli/libs/env" ) @@ -23,7 +26,11 @@ func (m *setVariables) Name() string { return "SetVariables" } -func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, name string) (dyn.Value, error) { +func getDefaultVariableFilePath(target string) string { + return ".databricks/bundle/" + target + "/variable-overrides.json" +} + +func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, name string, fileDefault dyn.Value) (dyn.Value, error) { // case: variable already has value initialized, so skip if variable.HasValue() { return v, nil @@ -49,6 +56,26 @@ func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, return v, nil } + // case: Set the variable to the default value from the variable file + if fileDefault.Kind() != dyn.KindInvalid && fileDefault.Kind() != dyn.KindNil { + hasComplexType := variable.IsComplex() + hasComplexValue := fileDefault.Kind() == dyn.KindMap || fileDefault.Kind() == dyn.KindSequence + + if hasComplexType && !hasComplexValue { + return dyn.InvalidValue, fmt.Errorf(`variable %s is of type complex, but the value in the variable file is not a complex type`, name) + } + if !hasComplexType && hasComplexValue { + return dyn.InvalidValue, fmt.Errorf(`variable %s is not of type complex, but the value in the variable file is a complex type`, name) + } + + v, err := dyn.Set(v, "value", fileDefault) + if err != nil { + return dyn.InvalidValue, fmt.Errorf(`failed to assign default value from variable file to variable %s with error: %v`, name, err) + } + + return v, nil + } + // case: Set the variable to its default value if variable.HasDefault() { vDefault, err := dyn.Get(v, "default") @@ -64,10 +91,43 @@ func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, } // We should have had a value to set for the variable at this point. - return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" flag or by setting the %s environment variable`, name, bundleVarPrefix+name) + return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done using "--var", by setting the %s environment variable, or in %s file`, name, bundleVarPrefix+name, getDefaultVariableFilePath("")) +} + +func readVariablesFromFile(b *bundle.Bundle) (dyn.Value, diag.Diagnostics) { + var diags diag.Diagnostics + + filePath := filepath.Join(b.BundleRootPath, getDefaultVariableFilePath(b.Config.Bundle.Target)) + if _, err := os.Stat(filePath); err != nil { + return dyn.InvalidValue, nil + } + + f, err := os.ReadFile(filePath) + if err != nil { + return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to read variables file: %w", err)) + } + + val, err := jsonloader.LoadJSON(f, filePath) + if err != nil { + return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to parse variables file %s: %w", filePath, err)) + } + + if val.Kind() != dyn.KindMap { + return dyn.InvalidValue, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("failed to parse variables file %s: invalid format", filePath), + Detail: "Variables file must be a JSON object with the following format:\n{\"var1\": \"value1\", \"var2\": \"value2\"}", + }) + } + + return val, nil } func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + defaults, diags := readVariablesFromFile(b) + if diags.HasError() { + return diags + } err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { return dyn.Map(v, "variables", dyn.Foreach(func(p dyn.Path, variable dyn.Value) (dyn.Value, error) { name := p[1].Key() @@ -76,9 +136,10 @@ func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos return dyn.InvalidValue, fmt.Errorf(`variable "%s" is not defined`, name) } - return setVariable(ctx, variable, v, name) + fileDefault, _ := dyn.Get(defaults, name) + return setVariable(ctx, variable, v, name, fileDefault) })) }) - return diag.FromErr(err) + return diags.Extend(diag.FromErr(err)) } diff --git a/bundle/config/mutator/set_variables_test.go b/bundle/config/mutator/set_variables_test.go index 07a5c8214..d904d5be3 100644 --- a/bundle/config/mutator/set_variables_test.go +++ b/bundle/config/mutator/set_variables_test.go @@ -25,7 +25,7 @@ func TestSetVariableFromProcessEnvVar(t *testing.T) { v, err := convert.FromTyped(variable, dyn.NilValue) require.NoError(t, err) - v, err = setVariable(context.Background(), v, &variable, "foo") + v, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue) require.NoError(t, err) err = convert.ToTyped(&variable, v) @@ -43,7 +43,7 @@ func TestSetVariableUsingDefaultValue(t *testing.T) { v, err := convert.FromTyped(variable, dyn.NilValue) require.NoError(t, err) - v, err = setVariable(context.Background(), v, &variable, "foo") + v, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue) require.NoError(t, err) err = convert.ToTyped(&variable, v) @@ -65,7 +65,7 @@ func TestSetVariableWhenAlreadyAValueIsAssigned(t *testing.T) { v, err := convert.FromTyped(variable, dyn.NilValue) require.NoError(t, err) - v, err = setVariable(context.Background(), v, &variable, "foo") + v, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue) require.NoError(t, err) err = convert.ToTyped(&variable, v) @@ -90,7 +90,7 @@ func TestSetVariableEnvVarValueDoesNotOverridePresetValue(t *testing.T) { v, err := convert.FromTyped(variable, dyn.NilValue) require.NoError(t, err) - v, err = setVariable(context.Background(), v, &variable, "foo") + v, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue) require.NoError(t, err) err = convert.ToTyped(&variable, v) @@ -107,8 +107,8 @@ func TestSetVariablesErrorsIfAValueCouldNotBeResolved(t *testing.T) { v, err := convert.FromTyped(variable, dyn.NilValue) require.NoError(t, err) - _, err = setVariable(context.Background(), v, &variable, "foo") - assert.ErrorContains(t, err, "no value assigned to required variable foo. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_foo environment variable") + _, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue) + assert.ErrorContains(t, err, "no value assigned to required variable foo. Assignment can be done using \"--var\", by setting the BUNDLE_VAR_foo environment variable, or in .databricks/bundle//variable-overrides.json file") } func TestSetVariablesMutator(t *testing.T) { @@ -157,6 +157,6 @@ func TestSetComplexVariablesViaEnvVariablesIsNotAllowed(t *testing.T) { v, err := convert.FromTyped(variable, dyn.NilValue) require.NoError(t, err) - _, err = setVariable(context.Background(), v, &variable, "foo") + _, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue) assert.ErrorContains(t, err, "setting via environment variables (BUNDLE_VAR_foo) is not supported for complex variable foo") } diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go index 95a68cfeb..d7f1cdede 100644 --- a/bundle/config/variable/variable.go +++ b/bundle/config/variable/variable.go @@ -36,11 +36,12 @@ type Variable struct { // This field stores the resolved value for the variable. The variable are // resolved in the following priority order (from highest to lowest) // - // 1. Command line flag. For example: `--var="foo=bar"` - // 2. Target variable. eg: BUNDLE_VAR_foo=bar - // 3. Default value as defined in the applicable environments block - // 4. Default value defined in variable definition - // 5. Throw error, since if no default value is defined, then the variable + // 1. Command line flag `--var="foo=bar"` + // 2. Environment variable. eg: BUNDLE_VAR_foo=bar + // 3. Load defaults from .databricks/bundle//variable-overrides.json + // 4. Default value as defined in the applicable targets block + // 5. Default value defined in variable definition + // 6. Throw error, since if no default value is defined, then the variable // is required Value VariableValue `json:"value,omitempty" bundle:"readonly"`