From b76eee0e8cf40199cd9fe154b79e101cb05aab9e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 15 Jan 2025 11:22:23 +0100 Subject: [PATCH] Migrate resolution tests to acceptance tests (#2143) --- .../resolve-nonstrings/databricks.yml | 23 ++++ .../variables/resolve-nonstrings/output.txt | 52 ++++++++ .../variables/resolve-nonstrings/script | 4 + .../resolve-vars-in-root-path/databricks.yml | 9 ++ .../resolve-vars-in-root-path/output.txt | 11 ++ .../resolve-vars-in-root-path/script | 1 + .../resolve_variable_references_test.go | 120 ------------------ 7 files changed, 100 insertions(+), 120 deletions(-) create mode 100644 acceptance/bundle/variables/resolve-nonstrings/databricks.yml create mode 100644 acceptance/bundle/variables/resolve-nonstrings/output.txt create mode 100644 acceptance/bundle/variables/resolve-nonstrings/script create mode 100644 acceptance/bundle/variables/resolve-vars-in-root-path/databricks.yml create mode 100644 acceptance/bundle/variables/resolve-vars-in-root-path/output.txt create mode 100644 acceptance/bundle/variables/resolve-vars-in-root-path/script diff --git a/acceptance/bundle/variables/resolve-nonstrings/databricks.yml b/acceptance/bundle/variables/resolve-nonstrings/databricks.yml new file mode 100644 index 000000000..a02c78a7e --- /dev/null +++ b/acceptance/bundle/variables/resolve-nonstrings/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: TestResolveVariableReferencesForPrimitiveNonStringFields + +variables: + no_alert_for_canceled_runs: {} + no_alert_for_skipped_runs: {} + min_workers: {} + max_workers: {} + spot_bid_max_price: {} + +resources: + jobs: + job1: + notification_settings: + no_alert_for_canceled_runs: ${var.no_alert_for_canceled_runs} + no_alert_for_skipped_runs: ${var.no_alert_for_skipped_runs} + tasks: + - new_cluster: + autoscale: + min_workers: ${var.min_workers} + max_workers: ${var.max_workers} + azure_attributes: + spot_bid_max_price: ${var.spot_bid_max_price} diff --git a/acceptance/bundle/variables/resolve-nonstrings/output.txt b/acceptance/bundle/variables/resolve-nonstrings/output.txt new file mode 100644 index 000000000..3a1eb9c47 --- /dev/null +++ b/acceptance/bundle/variables/resolve-nonstrings/output.txt @@ -0,0 +1,52 @@ +{ + "variables": { + "max_workers": { + "value": "2" + }, + "min_workers": { + "value": "1" + }, + "no_alert_for_canceled_runs": { + "value": "true" + }, + "no_alert_for_skipped_runs": { + "value": "false" + }, + "spot_bid_max_price": { + "value": "0.5" + } + }, + "jobs": { + "job1": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/TestResolveVariableReferencesForPrimitiveNonStringFields/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "notification_settings": { + "no_alert_for_canceled_runs": true, + "no_alert_for_skipped_runs": false + }, + "permissions": [], + "queue": { + "enabled": true + }, + "tags": {}, + "tasks": [ + { + "new_cluster": { + "autoscale": { + "max_workers": 2, + "min_workers": 1 + }, + "azure_attributes": { + "spot_bid_max_price": 0.5 + } + }, + "task_key": "" + } + ] + } + } +} diff --git a/acceptance/bundle/variables/resolve-nonstrings/script b/acceptance/bundle/variables/resolve-nonstrings/script new file mode 100644 index 000000000..cb9e45b61 --- /dev/null +++ b/acceptance/bundle/variables/resolve-nonstrings/script @@ -0,0 +1,4 @@ +export BUNDLE_VAR_no_alert_for_skipped_runs=false +export BUNDLE_VAR_max_workers=2 +export BUNDLE_VAR_min_workers=3 # shadowed by --var below +$CLI bundle validate -o json --var no_alert_for_canceled_runs=true --var min_workers=1 --var spot_bid_max_price=0.5 | jq '{ variables, jobs: .resources.jobs }' diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/databricks.yml b/acceptance/bundle/variables/resolve-vars-in-root-path/databricks.yml new file mode 100644 index 000000000..6a45de330 --- /dev/null +++ b/acceptance/bundle/variables/resolve-vars-in-root-path/databricks.yml @@ -0,0 +1,9 @@ +bundle: + name: TestResolveVariableReferencesToBundleVariables + +workspace: + root_path: "${bundle.name}/${var.foo}" + +variables: + foo: + value: "bar" diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt b/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt new file mode 100644 index 000000000..c56fbe415 --- /dev/null +++ b/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt @@ -0,0 +1,11 @@ +{ + "artifact_path": "TestResolveVariableReferencesToBundleVariables/bar/artifacts", + "current_user": { + "short_name": "$USERNAME", + "userName": "$USERNAME" + }, + "file_path": "TestResolveVariableReferencesToBundleVariables/bar/files", + "resource_path": "TestResolveVariableReferencesToBundleVariables/bar/resources", + "root_path": "TestResolveVariableReferencesToBundleVariables/bar", + "state_path": "TestResolveVariableReferencesToBundleVariables/bar/state" +} diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/script b/acceptance/bundle/variables/resolve-vars-in-root-path/script new file mode 100644 index 000000000..fefd9abe6 --- /dev/null +++ b/acceptance/bundle/variables/resolve-vars-in-root-path/script @@ -0,0 +1 @@ +$CLI bundle validate -o json | jq .workspace diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go index 18bb022aa..6bf84ae6d 100644 --- a/bundle/config/mutator/resolve_variable_references_test.go +++ b/bundle/config/mutator/resolve_variable_references_test.go @@ -13,129 +13,9 @@ import ( "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestResolveVariableReferencesToBundleVariables(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Bundle: config.Bundle{ - Name: "example", - }, - Workspace: config.Workspace{ - RootPath: "${bundle.name}/${var.foo}", - }, - Variables: map[string]*variable.Variable{ - "foo": { - Value: "bar", - }, - }, - }, - } - - // Apply with a valid prefix. This should change the workspace root path. - diags := bundle.Apply(context.Background(), b, ResolveVariableReferences("bundle", "variables")) - require.NoError(t, diags.Error()) - require.Equal(t, "example/bar", b.Config.Workspace.RootPath) -} - -func TestResolveVariableReferencesForPrimitiveNonStringFields(t *testing.T) { - var diags diag.Diagnostics - - b := &bundle.Bundle{ - Config: config.Root{ - Variables: map[string]*variable.Variable{ - "no_alert_for_canceled_runs": {}, - "no_alert_for_skipped_runs": {}, - "min_workers": {}, - "max_workers": {}, - "spot_bid_max_price": {}, - }, - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "job1": { - JobSettings: &jobs.JobSettings{ - NotificationSettings: &jobs.JobNotificationSettings{ - NoAlertForCanceledRuns: false, - NoAlertForSkippedRuns: false, - }, - Tasks: []jobs.Task{ - { - NewCluster: &compute.ClusterSpec{ - Autoscale: &compute.AutoScale{ - MinWorkers: 0, - MaxWorkers: 0, - }, - AzureAttributes: &compute.AzureAttributes{ - SpotBidMaxPrice: 0.0, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - ctx := context.Background() - - // Initialize the variables. - diags = bundle.ApplyFunc(ctx, b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.InitializeVariables([]string{ - "no_alert_for_canceled_runs=true", - "no_alert_for_skipped_runs=true", - "min_workers=1", - "max_workers=2", - "spot_bid_max_price=0.5", - }) - return diag.FromErr(err) - }) - require.NoError(t, diags.Error()) - - // Assign the variables to the dynamic configuration. - diags = bundle.ApplyFunc(ctx, b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - var p dyn.Path - var err error - - // Set the notification settings. - p = dyn.MustPathFromString("resources.jobs.job1.notification_settings") - v, err = dyn.SetByPath(v, p.Append(dyn.Key("no_alert_for_canceled_runs")), dyn.V("${var.no_alert_for_canceled_runs}")) - require.NoError(t, err) - v, err = dyn.SetByPath(v, p.Append(dyn.Key("no_alert_for_skipped_runs")), dyn.V("${var.no_alert_for_skipped_runs}")) - require.NoError(t, err) - - // Set the min and max workers. - p = dyn.MustPathFromString("resources.jobs.job1.tasks[0].new_cluster.autoscale") - v, err = dyn.SetByPath(v, p.Append(dyn.Key("min_workers")), dyn.V("${var.min_workers}")) - require.NoError(t, err) - v, err = dyn.SetByPath(v, p.Append(dyn.Key("max_workers")), dyn.V("${var.max_workers}")) - require.NoError(t, err) - - // Set the spot bid max price. - p = dyn.MustPathFromString("resources.jobs.job1.tasks[0].new_cluster.azure_attributes") - v, err = dyn.SetByPath(v, p.Append(dyn.Key("spot_bid_max_price")), dyn.V("${var.spot_bid_max_price}")) - require.NoError(t, err) - - return v, nil - }) - return diag.FromErr(err) - }) - require.NoError(t, diags.Error()) - - // Apply for the variable prefix. This should resolve the variables to their values. - diags = bundle.Apply(context.Background(), b, ResolveVariableReferences("variables")) - require.NoError(t, diags.Error()) - assert.True(t, b.Config.Resources.Jobs["job1"].JobSettings.NotificationSettings.NoAlertForCanceledRuns) - assert.True(t, b.Config.Resources.Jobs["job1"].JobSettings.NotificationSettings.NoAlertForSkippedRuns) - assert.Equal(t, 1, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.Autoscale.MinWorkers) - assert.Equal(t, 2, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.Autoscale.MaxWorkers) - assert.InDelta(t, 0.5, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.AzureAttributes.SpotBidMaxPrice, 0.0001) -} - func TestResolveComplexVariable(t *testing.T) { b := &bundle.Bundle{ Config: config.Root{