diff --git a/acceptance/bundle/variables/double_underscore/databricks.yml b/acceptance/bundle/variables/double_underscore/databricks.yml new file mode 100644 index 000000000..3bb15d42d --- /dev/null +++ b/acceptance/bundle/variables/double_underscore/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: double_underscore + +variables: + double__underscore: + description: "This is a variable with a double underscore" + default: "default" + +resources: + jobs: + test_job: + name: "test" + tasks: + - task_key: "test ${var.double__underscore}" diff --git a/acceptance/bundle/variables/double_underscore/output.txt b/acceptance/bundle/variables/double_underscore/output.txt new file mode 100644 index 000000000..45529038d --- /dev/null +++ b/acceptance/bundle/variables/double_underscore/output.txt @@ -0,0 +1,7 @@ + +>>> $CLI bundle validate -o json +[ + { + "task_key": "test default" + } +] diff --git a/acceptance/bundle/variables/double_underscore/script b/acceptance/bundle/variables/double_underscore/script new file mode 100644 index 000000000..a7394df77 --- /dev/null +++ b/acceptance/bundle/variables/double_underscore/script @@ -0,0 +1 @@ +trace $CLI bundle validate -o json | jq .resources.jobs.test_job.tasks diff --git a/integration/bundle/basic_test.go b/integration/bundle/basic_test.go index 79301b850..53f8e3ef6 100644 --- a/integration/bundle/basic_test.go +++ b/integration/bundle/basic_test.go @@ -6,7 +6,9 @@ import ( "testing" "github.com/databricks/cli/integration/internal/acc" + "github.com/databricks/cli/internal/testcli" "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/testdiff" "github.com/google/uuid" "github.com/stretchr/testify/require" ) @@ -35,3 +37,40 @@ func TestBasicBundleDeployWithFailOnActiveRuns(t *testing.T) { // deploy empty bundle again deployBundleWithFlags(t, ctx, root, []string{"--fail-on-active-runs"}) } + +func TestBasicBundleDeployWithDoubleUnderscoreVariables(t *testing.T) { + ctx, wt := acc.WorkspaceTest(t) + + nodeTypeId := testutil.GetCloud(t).NodeTypeID() + uniqueId := uuid.New().String() + root := initTestTemplate(t, ctx, "basic_with_variables", map[string]any{ + "unique_id": uniqueId, + "node_type_id": nodeTypeId, + "spark_version": defaultSparkVersion, + }) + + currentUser, err := wt.W.CurrentUser.Me(ctx) + require.NoError(t, err) + + ctx, replacements := testdiff.WithReplacementsMap(ctx) + replacements.Set(uniqueId, "$UNIQUE_PRJ") + replacements.Set(currentUser.UserName, "$USERNAME") + + t.Cleanup(func() { + destroyBundle(t, ctx, root) + }) + + testutil.Chdir(t, root) + testcli.AssertOutput( + t, + ctx, + []string{"bundle", "validate"}, + testutil.TestData("testdata/basic_with_variables/bundle_validate.txt"), + ) + testcli.AssertOutput( + t, + ctx, + []string{"bundle", "deploy", "--force-lock", "--auto-approve"}, + testutil.TestData("testdata/basic_with_variables/bundle_deploy.txt"), + ) +} diff --git a/integration/bundle/bundles/basic_with_variables/databricks_template_schema.json b/integration/bundle/bundles/basic_with_variables/databricks_template_schema.json new file mode 100644 index 000000000..41a723b0f --- /dev/null +++ b/integration/bundle/bundles/basic_with_variables/databricks_template_schema.json @@ -0,0 +1,21 @@ +{ + "properties": { + "unique_id": { + "type": "string", + "description": "Unique ID for job name" + }, + "spark_version": { + "type": "string", + "description": "Spark version used for job cluster" + }, + "node_type_id": { + "type": "string", + "description": "Node type id for job cluster" + }, + "root_path": { + "type": "string", + "description": "Root path to deploy bundle to", + "default": "" + } + } +} diff --git a/integration/bundle/bundles/basic_with_variables/template/databricks.yml.tmpl b/integration/bundle/bundles/basic_with_variables/template/databricks.yml.tmpl new file mode 100644 index 000000000..cb02c9e2f --- /dev/null +++ b/integration/bundle/bundles/basic_with_variables/template/databricks.yml.tmpl @@ -0,0 +1,32 @@ +bundle: + name: basic + +workspace: + {{ if .root_path }} + root_path: "{{.root_path}}/.bundle/{{.unique_id}}" + {{ else }} + root_path: "~/.bundle/{{.unique_id}}" + {{ end }} + +variables: + task__key: # Note: the variable has double underscore + default: my_notebook_task + +resources: + jobs: + foo__bar: # Note: the resource has double underscore to check that TF provider can use such names + name: test-job-basic-{{.unique_id}} + tasks: + - task_key: ${var.task__key} + new_cluster: + num_workers: 1 + spark_version: "{{.spark_version}}" + node_type_id: "{{.node_type_id}}" + spark_python_task: + python_file: ./hello_world.py + foo: + name: test-job-basic-ref-{{.unique_id}} + tasks: + - task_key: job_task + run_job_task: + job_id: ${resources.jobs.foo__bar.id} diff --git a/integration/bundle/bundles/basic_with_variables/template/hello_world.py b/integration/bundle/bundles/basic_with_variables/template/hello_world.py new file mode 100644 index 000000000..f301245e2 --- /dev/null +++ b/integration/bundle/bundles/basic_with_variables/template/hello_world.py @@ -0,0 +1 @@ +print("Hello World!") diff --git a/integration/bundle/testdata/basic_with_variables/bundle_deploy.txt b/integration/bundle/testdata/basic_with_variables/bundle_deploy.txt new file mode 100644 index 000000000..211164174 --- /dev/null +++ b/integration/bundle/testdata/basic_with_variables/bundle_deploy.txt @@ -0,0 +1,4 @@ +Uploading bundle files to /Workspace/Users/$USERNAME/.bundle/$UNIQUE_PRJ/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/integration/bundle/testdata/basic_with_variables/bundle_validate.txt b/integration/bundle/testdata/basic_with_variables/bundle_validate.txt new file mode 100644 index 000000000..dc9016a0f --- /dev/null +++ b/integration/bundle/testdata/basic_with_variables/bundle_validate.txt @@ -0,0 +1,7 @@ +Name: basic +Target: default +Workspace: + User: $USERNAME + Path: /Workspace/Users/$USERNAME/.bundle/$UNIQUE_PRJ + +Validation OK! diff --git a/libs/dyn/dynvar/ref.go b/libs/dyn/dynvar/ref.go index a28938823..ba397267a 100644 --- a/libs/dyn/dynvar/ref.go +++ b/libs/dyn/dynvar/ref.go @@ -1,12 +1,16 @@ package dynvar import ( + "fmt" "regexp" "github.com/databricks/cli/libs/dyn" ) -var re = regexp.MustCompile(`\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`) +var ( + baseVarDef = `[a-zA-Z]+([-_]*[a-zA-Z0-9]+)*` + re = regexp.MustCompile(fmt.Sprintf(`\$\{(%s(\.%s(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`, baseVarDef, baseVarDef)) +) // ref represents a variable reference. // It is a string [dyn.Value] contained in a larger [dyn.Value]. diff --git a/libs/dyn/dynvar/ref_test.go b/libs/dyn/dynvar/ref_test.go index 4110732f8..637ecb98e 100644 --- a/libs/dyn/dynvar/ref_test.go +++ b/libs/dyn/dynvar/ref_test.go @@ -15,9 +15,13 @@ func TestNewRefNoString(t *testing.T) { func TestNewRefValidPattern(t *testing.T) { for in, refs := range map[string][]string{ - "${hello_world.world_world}": {"hello_world.world_world"}, - "${helloworld.world-world}": {"helloworld.world-world"}, - "${hello-world.world-world}": {"hello-world.world-world"}, + "${hello_world.world_world}": {"hello_world.world_world"}, + "${helloworld.world-world}": {"helloworld.world-world"}, + "${hello-world.world-world}": {"hello-world.world-world"}, + "${hello_world.world__world}": {"hello_world.world__world"}, + "${hello_world.world--world}": {"hello_world.world--world"}, + "${hello_world.world-_world}": {"hello_world.world-_world"}, + "${hello_world.world_-world}": {"hello_world.world_-world"}, } { ref, ok := newRef(dyn.V(in)) require.True(t, ok, "should match valid pattern: %s", in) @@ -36,8 +40,6 @@ func TestNewRefInvalidPattern(t *testing.T) { "${_-_._-_.id}", // cannot use _- in sequence "${0helloworld.world-world}", // interpolated first section shouldn't start with number "${helloworld.9world-world}", // interpolated second section shouldn't start with number - "${a-a.a-_a-a.id}", // fails because of -_ in the second segment - "${a-a.a--a-a.id}", // fails because of -- in the second segment } for _, v := range invalid { _, ok := newRef(dyn.V(v))