diff --git a/acceptance/bundle/variables/empty/output.txt b/acceptance/bundle/variables/empty/output.txt index 261635920..589a0a502 100644 --- a/acceptance/bundle/variables/empty/output.txt +++ b/acceptance/bundle/variables/empty/output.txt @@ -1,4 +1,4 @@ -Error: no value assigned to required variable a. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_a environment variable +Error: no value assigned to required variable a. Assignment can be done through the "--var" or "--var-file" flag or by setting the BUNDLE_VAR_a environment variable Name: empty${var.a} Target: default diff --git a/acceptance/bundle/variables/env_overrides/output.txt b/acceptance/bundle/variables/env_overrides/output.txt index f42f82211..faa3505f3 100644 --- a/acceptance/bundle/variables/env_overrides/output.txt +++ b/acceptance/bundle/variables/env_overrides/output.txt @@ -9,7 +9,7 @@ "prod-a env-var-b" >>> errcode $CLI bundle validate -t env-missing-a-required-variable-assignment -Error: no value assigned to required variable b. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_b environment variable +Error: no value assigned to required variable b. Assignment can be done through the "--var" or "--var-file" flag or by setting the BUNDLE_VAR_b environment variable Name: test bundle Target: env-missing-a-required-variable-assignment diff --git a/acceptance/bundle/variables/flag_overrides/output.txt b/acceptance/bundle/variables/flag_overrides/output.txt deleted file mode 100644 index 549cf9e36..000000000 --- a/acceptance/bundle/variables/flag_overrides/output.txt +++ /dev/null @@ -1,8 +0,0 @@ -{ - "job_cluster_key": "mlops_stacks-cluster", - "new_cluster": { - "node_type_id": "Standard_DS3_v2", - "num_workers": 2 - } -} -"mlops_stacks-cluster" diff --git a/acceptance/bundle/variables/flag_overrides/script b/acceptance/bundle/variables/flag_overrides/script deleted file mode 100644 index bfdbdf427..000000000 --- a/acceptance/bundle/variables/flag_overrides/script +++ /dev/null @@ -1,5 +0,0 @@ -# variable file -$CLI bundle validate -o json --var-file=vars.json | jq .resources.jobs.job1.job_clusters[0] - -# variable flag -$CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster" | jq .resources.jobs.job1.job_clusters[0].job_cluster_key diff --git a/acceptance/bundle/variables/vanilla/output.txt b/acceptance/bundle/variables/vanilla/output.txt index 1d88bd060..36cf44b5a 100644 --- a/acceptance/bundle/variables/vanilla/output.txt +++ b/acceptance/bundle/variables/vanilla/output.txt @@ -3,7 +3,7 @@ "abc def" >>> errcode $CLI bundle validate -Error: no value assigned to required variable b. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_b environment variable +Error: no value assigned to required variable b. Assignment can be done through the "--var" or "--var-file" flag or by setting the BUNDLE_VAR_b environment variable Name: ${var.a} ${var.b} Target: default diff --git a/acceptance/bundle/variables/var_file_overrides/databricks.yml b/acceptance/bundle/variables/var_file_overrides/databricks.yml new file mode 100644 index 000000000..cf743a556 --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/databricks.yml @@ -0,0 +1,32 @@ +bundle: + name: TestResolveVariablesFromFile + +variables: + cluster: + type: "complex" + cluster_key: + cluster_workers: + +resources: + jobs: + job1: + job_clusters: + - job_cluster_key: ${var.cluster_key} + new_cluster: + node_type_id: "${var.cluster.node_type_id}" + num_workers: ${var.cluster_workers} + +targets: + with-defaults: + default: true + variables: + cluster_workers: + default: 1 + cluster: + type: "complex" + default: + node_type_id: "default" + cluster_key: + default: "default" + + without-defaults: diff --git a/acceptance/bundle/variables/var_file_overrides/output.txt b/acceptance/bundle/variables/var_file_overrides/output.txt new file mode 100644 index 000000000..1f6184397 --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/output.txt @@ -0,0 +1,126 @@ + +>>> $CLI bundle validate -o json --var-file=var_files/normal.json +{ + "job_cluster_key": "mlops_stacks-cluster", + "new_cluster": { + "node_type_id": "Standard_DS3_v2", + "num_workers": 2 + } +} + +>>> $CLI bundle validate -o json --var=cluster_key=mlops_stacks-cluster +{ + "job_cluster_key": "mlops_stacks-cluster", + "new_cluster": { + "node_type_id": "default", + "num_workers": 1 + } +} + +>>> errcode $CLI bundle validate -o json --var-file=var_files/normal.json --var=cluster_key=mlops_stacks-cluster +Error: cannot specify both --var and --var-file flags + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + +>>> errcode $CLI bundle validate -o json --var-file=var_files/not_found.json +Error: failed to read variables file: open var_files/not_found.json: no such file or directory + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + +>>> errcode $CLI bundle validate -o json --var-file=var_files/invalid_json.json +Error: failed to parse variables file: error decoding JSON at :0:0: invalid character 'o' in literal false (expecting 'a') + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + +>>> errcode $CLI bundle validate -o json --var-file=var_files/wrong_file_structure.json +Error: failed to parse variables file: var_files/wrong_file_structure.json:1:1: expected a map, found a sequence + in var_files/wrong_file_structure.json:1:1 + +Variables file must be a JSON object with the following format: +{"var1": "value1", "var2": "value2"} + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + +>>> errcode $CLI bundle validate -o json --var-file=var_files/undeclared.json +Error: variable undeclared_var has not been defined + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + +>>> errcode $CLI bundle validate -o json --var-file=var_files/complex_to_string.json +Error: expected a map to index "variables.cluster.value.node_type_id", found string + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + +>>> errcode $CLI bundle validate -o json --var-file=var_files/string_to_complex.json +Error: failed to assign map[node_type_id:Standard_DS3_v2] to cluster_key: variable type is not complex + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + +>>> errcode $CLI bundle validate -o json --target without-defaults --var-file=var_files/empty.json +Error: no value assigned to required variable cluster_workers. Assignment can be done through the "--var" or "--var-file" flag or by setting the BUNDLE_VAR_cluster_workers environment variable + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} diff --git a/acceptance/bundle/variables/var_file_overrides/script b/acceptance/bundle/variables/var_file_overrides/script new file mode 100644 index 000000000..147ce5acb --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/script @@ -0,0 +1,31 @@ +cluster_expr=".resources.jobs.job1.job_clusters[0]" + +# variable file +trace $CLI bundle validate -o json --var-file=var_files/normal.json | jq $cluster_expr + +# variable flag +trace $CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster" | jq $cluster_expr + +# both variable file and flag +trace errcode $CLI bundle validate -o json --var-file=var_files/normal.json --var="cluster_key=mlops_stacks-cluster" | jq $cluster_expr + +# file not found +trace errcode $CLI bundle validate -o json --var-file=var_files/not_found.json | jq $cluster_expr + +# file cannot be parsed +trace errcode $CLI bundle validate -o json --var-file=var_files/invalid_json.json | jq $cluster_expr + +# file has wrong structure +trace errcode $CLI bundle validate -o json --var-file=var_files/wrong_file_structure.json | jq $cluster_expr + +# file has variable name that is not defined +trace errcode $CLI bundle validate -o json --var-file=var_files/undeclared.json | jq $cluster_expr + +# file has variable name that is complex but default is string +trace errcode $CLI bundle validate -o json --var-file=var_files/complex_to_string.json | jq $cluster_expr + +# file has variable name that is string but default is complex +trace errcode $CLI bundle validate -o json --var-file=var_files/string_to_complex.json | jq $cluster_expr + +# variable is required but it's not provided in the file +trace errcode $CLI bundle validate -o json --target without-defaults --var-file=var_files/empty.json | jq $cluster_expr diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/complex_to_string.json b/acceptance/bundle/variables/var_file_overrides/var_files/complex_to_string.json new file mode 100644 index 000000000..1ea719446 --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/var_files/complex_to_string.json @@ -0,0 +1,3 @@ +{ + "cluster": "mlops_stacks-cluster" +} diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/empty.json b/acceptance/bundle/variables/var_file_overrides/var_files/empty.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/var_files/empty.json @@ -0,0 +1 @@ +{} diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/invalid_json.json b/acceptance/bundle/variables/var_file_overrides/var_files/invalid_json.json new file mode 100644 index 000000000..257cc5642 --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/var_files/invalid_json.json @@ -0,0 +1 @@ +foo diff --git a/acceptance/bundle/variables/flag_overrides/vars.json b/acceptance/bundle/variables/var_file_overrides/var_files/normal.json similarity index 100% rename from acceptance/bundle/variables/flag_overrides/vars.json rename to acceptance/bundle/variables/var_file_overrides/var_files/normal.json diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/string_to_complex.json b/acceptance/bundle/variables/var_file_overrides/var_files/string_to_complex.json new file mode 100644 index 000000000..602567a68 --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/var_files/string_to_complex.json @@ -0,0 +1,5 @@ +{ + "cluster_key": { + "node_type_id": "Standard_DS3_v2" + } +} diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/undeclared.json b/acceptance/bundle/variables/var_file_overrides/var_files/undeclared.json new file mode 100644 index 000000000..2a137d23d --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/var_files/undeclared.json @@ -0,0 +1,3 @@ +{ + "undeclared_var": 1 +} diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/wrong_file_structure.json b/acceptance/bundle/variables/var_file_overrides/var_files/wrong_file_structure.json new file mode 100644 index 000000000..de140ba36 --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/var_files/wrong_file_structure.json @@ -0,0 +1,3 @@ +[ + "foo" +] diff --git a/acceptance/bundle/variables/flag_overrides/databricks.yml b/acceptance/bundle/variables/var_flag/databricks.yml similarity index 78% rename from acceptance/bundle/variables/flag_overrides/databricks.yml rename to acceptance/bundle/variables/var_flag/databricks.yml index 41c106ac6..f9de114e5 100644 --- a/acceptance/bundle/variables/flag_overrides/databricks.yml +++ b/acceptance/bundle/variables/var_flag/databricks.yml @@ -1,13 +1,13 @@ bundle: - name: TestResolveVariablesFromFile + name: TestResolveVariablesFromFlag variables: cluster: type: "complex" default: - node_type_id: "unused" + node_type_id: "undefined" cluster_key: - default: "unused" + default: "undefined" cluster_workers: default: 1 diff --git a/acceptance/bundle/variables/var_flag/output.txt b/acceptance/bundle/variables/var_flag/output.txt new file mode 100644 index 000000000..b5390d371 --- /dev/null +++ b/acceptance/bundle/variables/var_flag/output.txt @@ -0,0 +1,3 @@ + +>>> $CLI bundle validate -o json --var=cluster_key=mlops_stacks-cluster +"mlops_stacks-cluster" diff --git a/acceptance/bundle/variables/var_flag/script b/acceptance/bundle/variables/var_flag/script new file mode 100644 index 000000000..66a1217d8 --- /dev/null +++ b/acceptance/bundle/variables/var_flag/script @@ -0,0 +1,2 @@ +# single flag +trace $CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster" | jq .resources.jobs.job1.job_clusters[0].job_cluster_key diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go index 9e9f2dcfe..7b16571e2 100644 --- a/bundle/config/mutator/set_variables.go +++ b/bundle/config/mutator/set_variables.go @@ -64,7 +64,7 @@ func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, } // We should have had a value to set for the variable at this point. - return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" flag or by setting the %s environment variable`, name, bundleVarPrefix+name) + return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" or "--var-file" flag or by setting the %s environment variable`, name, bundleVarPrefix+name) } func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {