From ec7808da34cb93c5a4c220edd1dc70b6e40ca4af Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 29 Jan 2025 14:38:28 +0100 Subject: [PATCH] Added support for double underscore variable references (#2203) ## Changes Added support for double underscore variable references. Previously we made this restriction stronger with no particular reason, TF provider supports multiple underscores and thus DABs should do as well. Fixes #1753 ## Tests Added acceptance and integration tests --- .../double_underscore/databricks.yml | 14 +++++++ .../variables/double_underscore/output.txt | 7 ++++ .../bundle/variables/double_underscore/script | 1 + integration/bundle/basic_test.go | 39 +++++++++++++++++++ .../databricks_template_schema.json | 21 ++++++++++ .../template/databricks.yml.tmpl | 32 +++++++++++++++ .../template/hello_world.py | 1 + .../basic_with_variables/bundle_deploy.txt | 4 ++ .../basic_with_variables/bundle_validate.txt | 7 ++++ libs/dyn/dynvar/ref.go | 6 ++- libs/dyn/dynvar/ref_test.go | 12 +++--- 11 files changed, 138 insertions(+), 6 deletions(-) create mode 100644 acceptance/bundle/variables/double_underscore/databricks.yml create mode 100644 acceptance/bundle/variables/double_underscore/output.txt create mode 100644 acceptance/bundle/variables/double_underscore/script create mode 100644 integration/bundle/bundles/basic_with_variables/databricks_template_schema.json create mode 100644 integration/bundle/bundles/basic_with_variables/template/databricks.yml.tmpl create mode 100644 integration/bundle/bundles/basic_with_variables/template/hello_world.py create mode 100644 integration/bundle/testdata/basic_with_variables/bundle_deploy.txt create mode 100644 integration/bundle/testdata/basic_with_variables/bundle_validate.txt diff --git a/acceptance/bundle/variables/double_underscore/databricks.yml b/acceptance/bundle/variables/double_underscore/databricks.yml new file mode 100644 index 000000000..3bb15d42d --- /dev/null +++ b/acceptance/bundle/variables/double_underscore/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: double_underscore + +variables: + double__underscore: + description: "This is a variable with a double underscore" + default: "default" + +resources: + jobs: + test_job: + name: "test" + tasks: + - task_key: "test ${var.double__underscore}" diff --git a/acceptance/bundle/variables/double_underscore/output.txt b/acceptance/bundle/variables/double_underscore/output.txt new file mode 100644 index 000000000..45529038d --- /dev/null +++ b/acceptance/bundle/variables/double_underscore/output.txt @@ -0,0 +1,7 @@ + +>>> $CLI bundle validate -o json +[ + { + "task_key": "test default" + } +] diff --git a/acceptance/bundle/variables/double_underscore/script b/acceptance/bundle/variables/double_underscore/script new file mode 100644 index 000000000..a7394df77 --- /dev/null +++ b/acceptance/bundle/variables/double_underscore/script @@ -0,0 +1 @@ +trace $CLI bundle validate -o json | jq .resources.jobs.test_job.tasks diff --git a/integration/bundle/basic_test.go b/integration/bundle/basic_test.go index 79301b850..53f8e3ef6 100644 --- a/integration/bundle/basic_test.go +++ b/integration/bundle/basic_test.go @@ -6,7 +6,9 @@ import ( "testing" "github.com/databricks/cli/integration/internal/acc" + "github.com/databricks/cli/internal/testcli" "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/testdiff" "github.com/google/uuid" "github.com/stretchr/testify/require" ) @@ -35,3 +37,40 @@ func TestBasicBundleDeployWithFailOnActiveRuns(t *testing.T) { // deploy empty bundle again deployBundleWithFlags(t, ctx, root, []string{"--fail-on-active-runs"}) } + +func TestBasicBundleDeployWithDoubleUnderscoreVariables(t *testing.T) { + ctx, wt := acc.WorkspaceTest(t) + + nodeTypeId := testutil.GetCloud(t).NodeTypeID() + uniqueId := uuid.New().String() + root := initTestTemplate(t, ctx, "basic_with_variables", map[string]any{ + "unique_id": uniqueId, + "node_type_id": nodeTypeId, + "spark_version": defaultSparkVersion, + }) + + currentUser, err := wt.W.CurrentUser.Me(ctx) + require.NoError(t, err) + + ctx, replacements := testdiff.WithReplacementsMap(ctx) + replacements.Set(uniqueId, "$UNIQUE_PRJ") + replacements.Set(currentUser.UserName, "$USERNAME") + + t.Cleanup(func() { + destroyBundle(t, ctx, root) + }) + + testutil.Chdir(t, root) + testcli.AssertOutput( + t, + ctx, + []string{"bundle", "validate"}, + testutil.TestData("testdata/basic_with_variables/bundle_validate.txt"), + ) + testcli.AssertOutput( + t, + ctx, + []string{"bundle", "deploy", "--force-lock", "--auto-approve"}, + testutil.TestData("testdata/basic_with_variables/bundle_deploy.txt"), + ) +} diff --git a/integration/bundle/bundles/basic_with_variables/databricks_template_schema.json b/integration/bundle/bundles/basic_with_variables/databricks_template_schema.json new file mode 100644 index 000000000..41a723b0f --- /dev/null +++ b/integration/bundle/bundles/basic_with_variables/databricks_template_schema.json @@ -0,0 +1,21 @@ +{ + "properties": { + "unique_id": { + "type": "string", + "description": "Unique ID for job name" + }, + "spark_version": { + "type": "string", + "description": "Spark version used for job cluster" + }, + "node_type_id": { + "type": "string", + "description": "Node type id for job cluster" + }, + "root_path": { + "type": "string", + "description": "Root path to deploy bundle to", + "default": "" + } + } +} diff --git a/integration/bundle/bundles/basic_with_variables/template/databricks.yml.tmpl b/integration/bundle/bundles/basic_with_variables/template/databricks.yml.tmpl new file mode 100644 index 000000000..cb02c9e2f --- /dev/null +++ b/integration/bundle/bundles/basic_with_variables/template/databricks.yml.tmpl @@ -0,0 +1,32 @@ +bundle: + name: basic + +workspace: + {{ if .root_path }} + root_path: "{{.root_path}}/.bundle/{{.unique_id}}" + {{ else }} + root_path: "~/.bundle/{{.unique_id}}" + {{ end }} + +variables: + task__key: # Note: the variable has double underscore + default: my_notebook_task + +resources: + jobs: + foo__bar: # Note: the resource has double underscore to check that TF provider can use such names + name: test-job-basic-{{.unique_id}} + tasks: + - task_key: ${var.task__key} + new_cluster: + num_workers: 1 + spark_version: "{{.spark_version}}" + node_type_id: "{{.node_type_id}}" + spark_python_task: + python_file: ./hello_world.py + foo: + name: test-job-basic-ref-{{.unique_id}} + tasks: + - task_key: job_task + run_job_task: + job_id: ${resources.jobs.foo__bar.id} diff --git a/integration/bundle/bundles/basic_with_variables/template/hello_world.py b/integration/bundle/bundles/basic_with_variables/template/hello_world.py new file mode 100644 index 000000000..f301245e2 --- /dev/null +++ b/integration/bundle/bundles/basic_with_variables/template/hello_world.py @@ -0,0 +1 @@ +print("Hello World!") diff --git a/integration/bundle/testdata/basic_with_variables/bundle_deploy.txt b/integration/bundle/testdata/basic_with_variables/bundle_deploy.txt new file mode 100644 index 000000000..211164174 --- /dev/null +++ b/integration/bundle/testdata/basic_with_variables/bundle_deploy.txt @@ -0,0 +1,4 @@ +Uploading bundle files to /Workspace/Users/$USERNAME/.bundle/$UNIQUE_PRJ/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/integration/bundle/testdata/basic_with_variables/bundle_validate.txt b/integration/bundle/testdata/basic_with_variables/bundle_validate.txt new file mode 100644 index 000000000..dc9016a0f --- /dev/null +++ b/integration/bundle/testdata/basic_with_variables/bundle_validate.txt @@ -0,0 +1,7 @@ +Name: basic +Target: default +Workspace: + User: $USERNAME + Path: /Workspace/Users/$USERNAME/.bundle/$UNIQUE_PRJ + +Validation OK! diff --git a/libs/dyn/dynvar/ref.go b/libs/dyn/dynvar/ref.go index a28938823..ba397267a 100644 --- a/libs/dyn/dynvar/ref.go +++ b/libs/dyn/dynvar/ref.go @@ -1,12 +1,16 @@ package dynvar import ( + "fmt" "regexp" "github.com/databricks/cli/libs/dyn" ) -var re = regexp.MustCompile(`\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`) +var ( + baseVarDef = `[a-zA-Z]+([-_]*[a-zA-Z0-9]+)*` + re = regexp.MustCompile(fmt.Sprintf(`\$\{(%s(\.%s(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`, baseVarDef, baseVarDef)) +) // ref represents a variable reference. // It is a string [dyn.Value] contained in a larger [dyn.Value]. diff --git a/libs/dyn/dynvar/ref_test.go b/libs/dyn/dynvar/ref_test.go index 4110732f8..637ecb98e 100644 --- a/libs/dyn/dynvar/ref_test.go +++ b/libs/dyn/dynvar/ref_test.go @@ -15,9 +15,13 @@ func TestNewRefNoString(t *testing.T) { func TestNewRefValidPattern(t *testing.T) { for in, refs := range map[string][]string{ - "${hello_world.world_world}": {"hello_world.world_world"}, - "${helloworld.world-world}": {"helloworld.world-world"}, - "${hello-world.world-world}": {"hello-world.world-world"}, + "${hello_world.world_world}": {"hello_world.world_world"}, + "${helloworld.world-world}": {"helloworld.world-world"}, + "${hello-world.world-world}": {"hello-world.world-world"}, + "${hello_world.world__world}": {"hello_world.world__world"}, + "${hello_world.world--world}": {"hello_world.world--world"}, + "${hello_world.world-_world}": {"hello_world.world-_world"}, + "${hello_world.world_-world}": {"hello_world.world_-world"}, } { ref, ok := newRef(dyn.V(in)) require.True(t, ok, "should match valid pattern: %s", in) @@ -36,8 +40,6 @@ func TestNewRefInvalidPattern(t *testing.T) { "${_-_._-_.id}", // cannot use _- in sequence "${0helloworld.world-world}", // interpolated first section shouldn't start with number "${helloworld.9world-world}", // interpolated second section shouldn't start with number - "${a-a.a-_a-a.id}", // fails because of -_ in the second segment - "${a-a.a--a-a.id}", // fails because of -- in the second segment } for _, v := range invalid { _, ok := newRef(dyn.V(v))