From 3a32c63919d06408d37557b651b6d55e4fc71247 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 21 Jan 2025 22:21:12 +0100 Subject: [PATCH 1/6] Add -inprocess mode for acceptance tests (#2184) ## Changes - If you pass -inprocess flag to acceptance tests, they will run in the same process as test itself. This enables debugging. - If you set singleTest variable on top of acceptance_test.go, you'll only run that test and with inprocess mode. This is intended for debugging in VSCode. - (minor) Converted KeepTmp to flag -keeptmp from env var KEEP_TMP for consistency with other flags. ## Tests - I verified that acceptance tests pass with -inprocess mode: `go test -inprocess < /dev/null | cat` - I verified that debugging in VSCode works: set a test name in singleTest variable, set breakpoints inside CLI and click "debug test" in VSCode. --- acceptance/acceptance_test.go | 65 +++++++++++++++++++++++++++---- acceptance/bin/callserver.py | 31 +++++++++++++++ acceptance/cmd_server_test.go | 73 +++++++++++++++++++++++++++++++++++ 3 files changed, 162 insertions(+), 7 deletions(-) create mode 100755 acceptance/bin/callserver.py create mode 100644 acceptance/cmd_server_test.go diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 12fe6536f..cfcb0d29f 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -3,6 +3,7 @@ package acceptance_test import ( "context" "errors" + "flag" "fmt" "io" "os" @@ -23,7 +24,22 @@ import ( "github.com/stretchr/testify/require" ) -var KeepTmp = os.Getenv("KEEP_TMP") != "" +var KeepTmp bool + +// In order to debug CLI running under acceptance test, set this to full subtest name, e.g. "bundle/variables/empty" +// Then install your breakpoints and click "debug test" near TestAccept in VSCODE. +// example: var singleTest = "bundle/variables/empty" +var singleTest = "" + +// If enabled, instead of compiling and running CLI externally, we'll start in-process server that accepts and runs +// CLI commands. The $CLI in test scripts is a helper that just forwards command-line arguments to this server (see bin/callserver.py). +// Also disables parallelism in tests. +var InprocessMode bool + +func init() { + flag.BoolVar(&InprocessMode, "inprocess", singleTest != "", "Run CLI in the same process as test (for debugging)") + flag.BoolVar(&KeepTmp, "keeptmp", false, "Do not delete TMP directory after run") +} const ( EntryPointScript = "script" @@ -38,6 +54,23 @@ var Scripts = map[string]bool{ } func TestAccept(t *testing.T) { + testAccept(t, InprocessMode, "") +} + +func TestInprocessMode(t *testing.T) { + if InprocessMode { + t.Skip("Already tested by TestAccept") + } + if runtime.GOOS == "windows" { + // - catalogs A catalog is the first layer of Unity Catalog’s three-level namespace. + // + catalogs A catalog is the first layer of Unity Catalog�s three-level namespace. + t.Skip("Fails on CI on unicode characters") + } + require.NotZero(t, testAccept(t, true, "help")) +} + +func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { + repls := testdiff.ReplacementsContext{} cwd, err := os.Getwd() require.NoError(t, err) @@ -50,16 +83,22 @@ func TestAccept(t *testing.T) { t.Logf("Writing coverage to %s", coverDir) } - execPath := BuildCLI(t, cwd, coverDir) - // $CLI is what test scripts are using + execPath := "" + + if InprocessMode { + cmdServer := StartCmdServer(t) + t.Setenv("CMD_SERVER_URL", cmdServer.URL) + execPath = filepath.Join(cwd, "bin", "callserver.py") + } else { + execPath = BuildCLI(t, cwd, coverDir) + } + t.Setenv("CLI", execPath) + repls.Set(execPath, "$CLI") // Make helper scripts available t.Setenv("PATH", fmt.Sprintf("%s%c%s", filepath.Join(cwd, "bin"), os.PathListSeparator, os.Getenv("PATH"))) - repls := testdiff.ReplacementsContext{} - repls.Set(execPath, "$CLI") - tempHomeDir := t.TempDir() repls.Set(tempHomeDir, "$TMPHOME") t.Logf("$TMPHOME=%v", tempHomeDir) @@ -95,13 +134,25 @@ func TestAccept(t *testing.T) { testDirs := getTests(t) require.NotEmpty(t, testDirs) + if singleTest != "" { + testDirs = slices.DeleteFunc(testDirs, func(n string) bool { + return n != singleTest + }) + require.NotEmpty(t, testDirs, "singleTest=%#v did not match any tests\n%#v", singleTest, testDirs) + } + for _, dir := range testDirs { testName := strings.ReplaceAll(dir, "\\", "/") t.Run(testName, func(t *testing.T) { - t.Parallel() + if !InprocessMode { + t.Parallel() + } + runTest(t, dir, coverDir, repls.Clone()) }) } + + return len(testDirs) } func getTests(t *testing.T) []string { diff --git a/acceptance/bin/callserver.py b/acceptance/bin/callserver.py new file mode 100755 index 000000000..294ef8fdb --- /dev/null +++ b/acceptance/bin/callserver.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +import sys +import os +import json +import urllib.request +from urllib.parse import urlencode + +env = {} +for key, value in os.environ.items(): + if len(value) > 10_000: + sys.stderr.write(f"Dropping key={key} value len={len(value)}\n") + continue + env[key] = value + +q = { + "args": " ".join(sys.argv[1:]), + "cwd": os.getcwd(), + "env": json.dumps(env), +} + +url = os.environ["CMD_SERVER_URL"] + "/?" + urlencode(q) +if len(url) > 100_000: + sys.exit("url too large") + +resp = urllib.request.urlopen(url) +assert resp.status == 200, (resp.status, resp.url, resp.headers) +result = json.load(resp) +sys.stderr.write(result["stderr"]) +sys.stdout.write(result["stdout"]) +exitcode = int(result["exitcode"]) +sys.exit(exitcode) diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go new file mode 100644 index 000000000..28feec1bd --- /dev/null +++ b/acceptance/cmd_server_test.go @@ -0,0 +1,73 @@ +package acceptance_test + +import ( + "encoding/json" + "net/http" + "os" + "strings" + "testing" + + "github.com/databricks/cli/internal/testcli" + "github.com/stretchr/testify/require" +) + +func StartCmdServer(t *testing.T) *TestServer { + server := StartServer(t) + server.Handle("/", func(r *http.Request) (any, error) { + q := r.URL.Query() + args := strings.Split(q.Get("args"), " ") + + var env map[string]string + require.NoError(t, json.Unmarshal([]byte(q.Get("env")), &env)) + + for key, val := range env { + defer Setenv(t, key, val)() + } + + defer Chdir(t, q.Get("cwd"))() + + c := testcli.NewRunner(t, r.Context(), args...) + c.Verbose = false + stdout, stderr, err := c.Run() + result := map[string]any{ + "stdout": stdout.String(), + "stderr": stderr.String(), + } + exitcode := 0 + if err != nil { + exitcode = 1 + } + result["exitcode"] = exitcode + return result, nil + }) + return server +} + +// Chdir variant that is intended to be used with defer so that it can switch back before function ends. +// This is unlike testutil.Chdir which switches back only when tests end. +func Chdir(t *testing.T, cwd string) func() { + require.NotEmpty(t, cwd) + prevDir, err := os.Getwd() + require.NoError(t, err) + err = os.Chdir(cwd) + require.NoError(t, err) + return func() { + _ = os.Chdir(prevDir) + } +} + +// Setenv variant that is intended to be used with defer so that it can switch back before function ends. +// This is unlike t.Setenv which switches back only when tests end. +func Setenv(t *testing.T, key, value string) func() { + prevVal, exists := os.LookupEnv(key) + + require.NoError(t, os.Setenv(key, value)) + + return func() { + if exists { + _ = os.Setenv(key, prevVal) + } else { + _ = os.Unsetenv(key) + } + } +} From fde30ff1ab5bf504e326f58adfd9f429d88b3b6d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 22 Jan 2025 11:17:45 +0100 Subject: [PATCH 2/6] Add a test for sync root outside of git root (#2202) - Move acceptance/bundle/sync-paths-dotdot test to acceptance/bundle/syncroot/dotdot-notgit - Add new test acceptance/bundle/syncroot/dotdot-git Fix replacer to work with this test and on Windows: - Make PATH work on Windows by using EvalSymlinks. - Make concatenated path match within JSON but stripping quotes. --- acceptance/acceptance_test.go | 12 +++--- acceptance/bundle/sync-paths-dotdot/script | 1 - .../dotdot-git}/databricks.yml | 0 .../dotdot-git}/output.txt | 2 +- acceptance/bundle/syncroot/dotdot-git/script | 6 +++ .../syncroot/dotdot-nogit/databricks.yml | 5 +++ .../bundle/syncroot/dotdot-nogit/output.txt | 11 ++++++ .../bundle/syncroot/dotdot-nogit/script | 2 + libs/testdiff/replacement.go | 39 ++++++++++++++++++- 9 files changed, 69 insertions(+), 9 deletions(-) delete mode 100644 acceptance/bundle/sync-paths-dotdot/script rename acceptance/bundle/{sync-paths-dotdot => syncroot/dotdot-git}/databricks.yml (100%) rename acceptance/bundle/{sync-paths-dotdot => syncroot/dotdot-git}/output.txt (69%) create mode 100644 acceptance/bundle/syncroot/dotdot-git/script create mode 100644 acceptance/bundle/syncroot/dotdot-nogit/databricks.yml create mode 100644 acceptance/bundle/syncroot/dotdot-nogit/output.txt create mode 100644 acceptance/bundle/syncroot/dotdot-nogit/script diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index cfcb0d29f..9a4564ffa 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -188,12 +188,12 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont tmpDir = t.TempDir() } - repls.Set("/private"+tmpDir, "$TMPDIR") - repls.Set("/private"+filepath.Dir(tmpDir), "$TMPPARENT") - repls.Set("/private"+filepath.Dir(filepath.Dir(tmpDir)), "$TMPGPARENT") - repls.Set(tmpDir, "$TMPDIR") - repls.Set(filepath.Dir(tmpDir), "$TMPPARENT") - repls.Set(filepath.Dir(filepath.Dir(tmpDir)), "$TMPGPARENT") + // Converts C:\Users\DENIS~1.BIL -> C:\Users\denis.bilenko + tmpDirEvalled, err1 := filepath.EvalSymlinks(tmpDir) + if err1 == nil && tmpDirEvalled != tmpDir { + repls.SetPathWithParents(tmpDirEvalled, "$TMPDIR") + } + repls.SetPathWithParents(tmpDir, "$TMPDIR") scriptContents := readMergedScriptContents(t, dir) testutil.WriteFile(t, filepath.Join(tmpDir, EntryPointScript), scriptContents) diff --git a/acceptance/bundle/sync-paths-dotdot/script b/acceptance/bundle/sync-paths-dotdot/script deleted file mode 100644 index 72555b332..000000000 --- a/acceptance/bundle/sync-paths-dotdot/script +++ /dev/null @@ -1 +0,0 @@ -$CLI bundle validate diff --git a/acceptance/bundle/sync-paths-dotdot/databricks.yml b/acceptance/bundle/syncroot/dotdot-git/databricks.yml similarity index 100% rename from acceptance/bundle/sync-paths-dotdot/databricks.yml rename to acceptance/bundle/syncroot/dotdot-git/databricks.yml diff --git a/acceptance/bundle/sync-paths-dotdot/output.txt b/acceptance/bundle/syncroot/dotdot-git/output.txt similarity index 69% rename from acceptance/bundle/sync-paths-dotdot/output.txt rename to acceptance/bundle/syncroot/dotdot-git/output.txt index 11db3e9ee..f1dc5fb01 100644 --- a/acceptance/bundle/sync-paths-dotdot/output.txt +++ b/acceptance/bundle/syncroot/dotdot-git/output.txt @@ -1,4 +1,4 @@ -Error: path "$TMPPARENT" is not within repository root "$TMPDIR" +Error: path "$TMPDIR" is not within repository root "$TMPDIR/myrepo" Name: test-bundle Target: default diff --git a/acceptance/bundle/syncroot/dotdot-git/script b/acceptance/bundle/syncroot/dotdot-git/script new file mode 100644 index 000000000..0706a1d5e --- /dev/null +++ b/acceptance/bundle/syncroot/dotdot-git/script @@ -0,0 +1,6 @@ +# This should error, we do not allow syncroot outside of git repo. +mkdir myrepo +cd myrepo +cp ../databricks.yml . +git-repo-init +$CLI bundle validate | sed 's/\\\\/\//g' diff --git a/acceptance/bundle/syncroot/dotdot-nogit/databricks.yml b/acceptance/bundle/syncroot/dotdot-nogit/databricks.yml new file mode 100644 index 000000000..7215ffea2 --- /dev/null +++ b/acceptance/bundle/syncroot/dotdot-nogit/databricks.yml @@ -0,0 +1,5 @@ +bundle: + name: test-bundle +sync: + paths: + - .. diff --git a/acceptance/bundle/syncroot/dotdot-nogit/output.txt b/acceptance/bundle/syncroot/dotdot-nogit/output.txt new file mode 100644 index 000000000..34059e276 --- /dev/null +++ b/acceptance/bundle/syncroot/dotdot-nogit/output.txt @@ -0,0 +1,11 @@ +Error: path "$TMPDIR_PARENT" is not within repository root "$TMPDIR" + +Name: test-bundle +Target: default +Workspace: + User: $USERNAME + Path: /Workspace/Users/$USERNAME/.bundle/test-bundle/default + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/syncroot/dotdot-nogit/script b/acceptance/bundle/syncroot/dotdot-nogit/script new file mode 100644 index 000000000..d3388903e --- /dev/null +++ b/acceptance/bundle/syncroot/dotdot-nogit/script @@ -0,0 +1,2 @@ +# This should not error, syncroot can be outside bundle root. +$CLI bundle validate diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go index 207f425aa..ca76b159c 100644 --- a/libs/testdiff/replacement.go +++ b/libs/testdiff/replacement.go @@ -3,7 +3,9 @@ package testdiff import ( "encoding/json" "fmt" + "path/filepath" "regexp" + "runtime" "slices" "strings" @@ -74,13 +76,48 @@ func (r *ReplacementsContext) Set(old, new string) { if err == nil { encodedOld, err := json.Marshal(old) if err == nil { - r.appendLiteral(string(encodedOld), string(encodedNew)) + r.appendLiteral(trimQuotes(string(encodedOld)), trimQuotes(string(encodedNew))) } } r.appendLiteral(old, new) } +func trimQuotes(s string) string { + if len(s) > 0 && s[0] == '"' { + s = s[1:] + } + if len(s) > 0 && s[len(s)-1] == '"' { + s = s[:len(s)-1] + } + return s +} + +func (r *ReplacementsContext) SetPath(old, new string) { + r.Set(old, new) + + if runtime.GOOS != "windows" { + return + } + + // Support both forward and backward slashes + m1 := strings.ReplaceAll(old, "\\", "/") + if m1 != old { + r.Set(m1, new) + } + + m2 := strings.ReplaceAll(old, "/", "\\") + if m2 != old && m2 != m1 { + r.Set(m2, new) + } +} + +func (r *ReplacementsContext) SetPathWithParents(old, new string) { + r.SetPath(old, new) + r.SetPath(filepath.Dir(old), new+"_PARENT") + r.SetPath(filepath.Dir(filepath.Dir(old)), new+"_GPARENT") +} + func PrepareReplacementsWorkspaceClient(t testutil.TestingT, r *ReplacementsContext, w *databricks.WorkspaceClient) { t.Helper() // in some clouds (gcp) w.Config.Host includes "https://" prefix in others it's really just a host (azure) From c224be5c1f9c0d082c6453031f740158ffaab91c Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Wed, 22 Jan 2025 11:30:17 +0100 Subject: [PATCH 3/6] Allow using variables in enum fields (#2199) ## Changes It is possible to pass variable to enum fields but json-schema doesn't accept it. This PR adds `oneOf` for enum types that includes `${var-*}` pattern ## Tests Manually checked in VSCode --- bundle/internal/schema/main.go | 13 + bundle/internal/schema/testdata/pass/job.yml | 4 + bundle/schema/embed_test.go | 8 +- bundle/schema/jsonschema.json | 916 +++++++++++++------ 4 files changed, 663 insertions(+), 278 deletions(-) diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 77927a966..39b859656 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -40,6 +40,19 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema. } } + // Allows using variables in enum fields + if s.Type == jsonschema.StringType && s.Enum != nil { + return jsonschema.Schema{ + OneOf: []jsonschema.Schema{ + s, + { + Type: jsonschema.StringType, + Pattern: interpolationPattern("var"), + }, + }, + } + } + switch s.Type { case jsonschema.ArrayType, jsonschema.ObjectType: // arrays and objects can have complex variable values specified. diff --git a/bundle/internal/schema/testdata/pass/job.yml b/bundle/internal/schema/testdata/pass/job.yml index e13a52c03..ec447ba39 100644 --- a/bundle/internal/schema/testdata/pass/job.yml +++ b/bundle/internal/schema/testdata/pass/job.yml @@ -13,6 +13,8 @@ variables: simplevar: default: true description: "simplevar description" + schedule_status: + default: "PAUSED" complexvar: default: @@ -42,6 +44,8 @@ resources: dependencies: - python=3.7 client: "myclient" + trigger: + pause_status: ${var.schedule_status} tags: foo: bar bar: baz diff --git a/bundle/schema/embed_test.go b/bundle/schema/embed_test.go index 59f1458cb..03d2165e4 100644 --- a/bundle/schema/embed_test.go +++ b/bundle/schema/embed_test.go @@ -59,8 +59,8 @@ func TestJsonSchema(t *testing.T) { } providers := walk(s.Definitions, "github.com", "databricks", "databricks-sdk-go", "service", "jobs.GitProvider") - assert.Contains(t, providers.Enum, "gitHub") - assert.Contains(t, providers.Enum, "bitbucketCloud") - assert.Contains(t, providers.Enum, "gitHubEnterprise") - assert.Contains(t, providers.Enum, "bitbucketServer") + assert.Contains(t, providers.OneOf[0].Enum, "gitHub") + assert.Contains(t, providers.OneOf[0].Enum, "bitbucketCloud") + assert.Contains(t, providers.OneOf[0].Enum, "gitHubEnterprise") + assert.Contains(t, providers.OneOf[0].Enum, "bitbucketServer") } diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 57b0cd231..b3158792c 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1663,19 +1663,35 @@ ] }, "apps.AppDeploymentMode": { - "type": "string", - "enum": [ - "SNAPSHOT", - "AUTO_SYNC" + "oneOf": [ + { + "type": "string", + "enum": [ + "SNAPSHOT", + "AUTO_SYNC" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "apps.AppDeploymentState": { - "type": "string", - "enum": [ - "SUCCEEDED", - "FAILED", - "IN_PROGRESS", - "CANCELLED" + "oneOf": [ + { + "type": "string", + "enum": [ + "SUCCEEDED", + "FAILED", + "IN_PROGRESS", + "CANCELLED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "apps.AppDeploymentStatus": { @@ -1758,12 +1774,20 @@ ] }, "apps.AppResourceJobJobPermission": { - "type": "string", - "enum": [ - "CAN_MANAGE", - "IS_OWNER", - "CAN_MANAGE_RUN", - "CAN_VIEW" + "oneOf": [ + { + "type": "string", + "enum": [ + "CAN_MANAGE", + "IS_OWNER", + "CAN_MANAGE_RUN", + "CAN_VIEW" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "apps.AppResourceSecret": { @@ -1795,12 +1819,20 @@ ] }, "apps.AppResourceSecretSecretPermission": { - "type": "string", - "description": "Permission to grant on the secret scope. Supported permissions are: \"READ\", \"WRITE\", \"MANAGE\".", - "enum": [ - "READ", - "WRITE", - "MANAGE" + "oneOf": [ + { + "type": "string", + "description": "Permission to grant on the secret scope. Supported permissions are: \"READ\", \"WRITE\", \"MANAGE\".", + "enum": [ + "READ", + "WRITE", + "MANAGE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "apps.AppResourceServingEndpoint": { @@ -1828,11 +1860,19 @@ ] }, "apps.AppResourceServingEndpointServingEndpointPermission": { - "type": "string", - "enum": [ - "CAN_MANAGE", - "CAN_QUERY", - "CAN_VIEW" + "oneOf": [ + { + "type": "string", + "enum": [ + "CAN_MANAGE", + "CAN_QUERY", + "CAN_VIEW" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "apps.AppResourceSqlWarehouse": { @@ -1860,20 +1900,36 @@ ] }, "apps.AppResourceSqlWarehouseSqlWarehousePermission": { - "type": "string", - "enum": [ - "CAN_MANAGE", - "CAN_USE", - "IS_OWNER" + "oneOf": [ + { + "type": "string", + "enum": [ + "CAN_MANAGE", + "CAN_USE", + "IS_OWNER" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "apps.ApplicationState": { - "type": "string", - "enum": [ - "DEPLOYING", - "RUNNING", - "CRASHED", - "UNAVAILABLE" + "oneOf": [ + { + "type": "string", + "enum": [ + "DEPLOYING", + "RUNNING", + "CRASHED", + "UNAVAILABLE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "apps.ApplicationStatus": { @@ -1897,15 +1953,23 @@ ] }, "apps.ComputeState": { - "type": "string", - "enum": [ - "ERROR", - "DELETING", - "STARTING", - "STOPPING", - "UPDATING", - "STOPPED", - "ACTIVE" + "oneOf": [ + { + "type": "string", + "enum": [ + "ERROR", + "DELETING", + "STARTING", + "STOPPING", + "UPDATING", + "STOPPED", + "ACTIVE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "apps.ComputeStatus": { @@ -1959,11 +2023,19 @@ ] }, "catalog.MonitorCronSchedulePauseStatus": { - "type": "string", - "description": "Read only field that indicates whether a schedule is paused or not.", - "enum": [ - "UNPAUSED", - "PAUSED" + "oneOf": [ + { + "type": "string", + "description": "Read only field that indicates whether a schedule is paused or not.", + "enum": [ + "UNPAUSED", + "PAUSED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "catalog.MonitorDataClassificationConfig": { @@ -2052,11 +2124,19 @@ ] }, "catalog.MonitorInferenceLogProblemType": { - "type": "string", - "description": "Problem type the model aims to solve. Determines the type of model-quality metrics that will be computed.", - "enum": [ - "PROBLEM_TYPE_CLASSIFICATION", - "PROBLEM_TYPE_REGRESSION" + "oneOf": [ + { + "type": "string", + "description": "Problem type the model aims to solve. Determines the type of model-quality metrics that will be computed.", + "enum": [ + "PROBLEM_TYPE_CLASSIFICATION", + "PROBLEM_TYPE_REGRESSION" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "catalog.MonitorMetric": { @@ -2101,12 +2181,20 @@ ] }, "catalog.MonitorMetricType": { - "type": "string", - "description": "Can only be one of ``\"CUSTOM_METRIC_TYPE_AGGREGATE\"``, ``\"CUSTOM_METRIC_TYPE_DERIVED\"``, or ``\"CUSTOM_METRIC_TYPE_DRIFT\"``.\nThe ``\"CUSTOM_METRIC_TYPE_AGGREGATE\"`` and ``\"CUSTOM_METRIC_TYPE_DERIVED\"`` metrics\nare computed on a single table, whereas the ``\"CUSTOM_METRIC_TYPE_DRIFT\"`` compare metrics across\nbaseline and input table, or across the two consecutive time windows.\n- CUSTOM_METRIC_TYPE_AGGREGATE: only depend on the existing columns in your table\n- CUSTOM_METRIC_TYPE_DERIVED: depend on previously computed aggregate metrics\n- CUSTOM_METRIC_TYPE_DRIFT: depend on previously computed aggregate or derived metrics\n", - "enum": [ - "CUSTOM_METRIC_TYPE_AGGREGATE", - "CUSTOM_METRIC_TYPE_DERIVED", - "CUSTOM_METRIC_TYPE_DRIFT" + "oneOf": [ + { + "type": "string", + "description": "Can only be one of ``\"CUSTOM_METRIC_TYPE_AGGREGATE\"``, ``\"CUSTOM_METRIC_TYPE_DERIVED\"``, or ``\"CUSTOM_METRIC_TYPE_DRIFT\"``.\nThe ``\"CUSTOM_METRIC_TYPE_AGGREGATE\"`` and ``\"CUSTOM_METRIC_TYPE_DERIVED\"`` metrics\nare computed on a single table, whereas the ``\"CUSTOM_METRIC_TYPE_DRIFT\"`` compare metrics across\nbaseline and input table, or across the two consecutive time windows.\n- CUSTOM_METRIC_TYPE_AGGREGATE: only depend on the existing columns in your table\n- CUSTOM_METRIC_TYPE_DERIVED: depend on previously computed aggregate metrics\n- CUSTOM_METRIC_TYPE_DRIFT: depend on previously computed aggregate or derived metrics\n", + "enum": [ + "CUSTOM_METRIC_TYPE_AGGREGATE", + "CUSTOM_METRIC_TYPE_DERIVED", + "CUSTOM_METRIC_TYPE_DRIFT" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "catalog.MonitorNotifications": { @@ -2170,10 +2258,18 @@ ] }, "catalog.VolumeType": { - "type": "string", - "enum": [ - "EXTERNAL", - "MANAGED" + "oneOf": [ + { + "type": "string", + "enum": [ + "EXTERNAL", + "MANAGED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "compute.Adlsgen2Info": { @@ -2272,12 +2368,20 @@ ] }, "compute.AwsAvailability": { - "type": "string", - "description": "Availability type used for all subsequent nodes past the `first_on_demand` ones.\n\nNote: If `first_on_demand` is zero, this availability type will be used for the entire cluster.\n", - "enum": [ - "SPOT", - "ON_DEMAND", - "SPOT_WITH_FALLBACK" + "oneOf": [ + { + "type": "string", + "description": "Availability type used for all subsequent nodes past the `first_on_demand` ones.\n\nNote: If `first_on_demand` is zero, this availability type will be used for the entire cluster.\n", + "enum": [ + "SPOT", + "ON_DEMAND", + "SPOT_WITH_FALLBACK" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "compute.AzureAttributes": { @@ -2310,12 +2414,20 @@ ] }, "compute.AzureAvailability": { - "type": "string", - "description": "Availability type used for all subsequent nodes past the `first_on_demand` ones.\nNote: If `first_on_demand` is zero (which only happens on pool clusters), this availability\ntype will be used for the entire cluster.", - "enum": [ - "SPOT_AZURE", - "ON_DEMAND_AZURE", - "SPOT_WITH_FALLBACK_AZURE" + "oneOf": [ + { + "type": "string", + "description": "Availability type used for all subsequent nodes past the `first_on_demand` ones.\nNote: If `first_on_demand` is zero (which only happens on pool clusters), this availability\ntype will be used for the entire cluster.", + "enum": [ + "SPOT_AZURE", + "ON_DEMAND_AZURE", + "SPOT_WITH_FALLBACK_AZURE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "compute.ClientsTypes": { @@ -2492,19 +2604,27 @@ ] }, "compute.DataSecurityMode": { - "type": "string", - "description": "Data security mode decides what data governance model to use when accessing data\nfrom a cluster.\n\nThe following modes can only be used with `kind`.\n* `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration.\n* `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`.\n* `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`.\n\nThe following modes can be used regardless of `kind`.\n* `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode.\n* `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode.\n* `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.\n\nThe following modes are deprecated starting with Databricks Runtime 15.0 and\nwill be removed for future Databricks Runtime versions:\n\n* `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters.\n* `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters.\n* `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters.\n* `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled.\n", - "enum": [ - "DATA_SECURITY_MODE_AUTO", - "DATA_SECURITY_MODE_STANDARD", - "DATA_SECURITY_MODE_DEDICATED", - "NONE", - "SINGLE_USER", - "USER_ISOLATION", - "LEGACY_TABLE_ACL", - "LEGACY_PASSTHROUGH", - "LEGACY_SINGLE_USER", - "LEGACY_SINGLE_USER_STANDARD" + "oneOf": [ + { + "type": "string", + "description": "Data security mode decides what data governance model to use when accessing data\nfrom a cluster.\n\nThe following modes can only be used with `kind`.\n* `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration.\n* `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`.\n* `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`.\n\nThe following modes can be used regardless of `kind`.\n* `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode.\n* `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode.\n* `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.\n\nThe following modes are deprecated starting with Databricks Runtime 15.0 and\nwill be removed for future Databricks Runtime versions:\n\n* `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters.\n* `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters.\n* `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters.\n* `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled.\n", + "enum": [ + "DATA_SECURITY_MODE_AUTO", + "DATA_SECURITY_MODE_STANDARD", + "DATA_SECURITY_MODE_DEDICATED", + "NONE", + "SINGLE_USER", + "USER_ISOLATION", + "LEGACY_TABLE_ACL", + "LEGACY_PASSTHROUGH", + "LEGACY_SINGLE_USER", + "LEGACY_SINGLE_USER_STANDARD" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "compute.DbfsStorageInfo": { @@ -2572,11 +2692,19 @@ ] }, "compute.EbsVolumeType": { - "type": "string", - "description": "The type of EBS volumes that will be launched with this cluster.", - "enum": [ - "GENERAL_PURPOSE_SSD", - "THROUGHPUT_OPTIMIZED_HDD" + "oneOf": [ + { + "type": "string", + "description": "The type of EBS volumes that will be launched with this cluster.", + "enum": [ + "GENERAL_PURPOSE_SSD", + "THROUGHPUT_OPTIMIZED_HDD" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "compute.Environment": { @@ -2643,12 +2771,20 @@ ] }, "compute.GcpAvailability": { - "type": "string", - "description": "This field determines whether the instance pool will contain preemptible\nVMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable.", - "enum": [ - "PREEMPTIBLE_GCP", - "ON_DEMAND_GCP", - "PREEMPTIBLE_WITH_FALLBACK_GCP" + "oneOf": [ + { + "type": "string", + "description": "This field determines whether the instance pool will contain preemptible\nVMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable.", + "enum": [ + "PREEMPTIBLE_GCP", + "ON_DEMAND_GCP", + "PREEMPTIBLE_WITH_FALLBACK_GCP" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "compute.GcsStorageInfo": { @@ -2882,12 +3018,20 @@ ] }, "compute.RuntimeEngine": { - "type": "string", - "description": "Determines the cluster's runtime engine, either standard or Photon.\n\nThis field is not compatible with legacy `spark_version` values that contain `-photon-`.\nRemove `-photon-` from the `spark_version` and set `runtime_engine` to `PHOTON`.\n\nIf left unspecified, the runtime engine defaults to standard unless the spark_version\ncontains -photon-, in which case Photon will be used.\n", - "enum": [ - "NULL", - "STANDARD", - "PHOTON" + "oneOf": [ + { + "type": "string", + "description": "Determines the cluster's runtime engine, either standard or Photon.\n\nThis field is not compatible with legacy `spark_version` values that contain `-photon-`.\nRemove `-photon-` from the `spark_version` and set `runtime_engine` to `PHOTON`.\n\nIf left unspecified, the runtime engine defaults to standard unless the spark_version\ncontains -photon-, in which case Photon will be used.\n", + "enum": [ + "NULL", + "STANDARD", + "PHOTON" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "compute.S3StorageInfo": { @@ -2999,10 +3143,18 @@ ] }, "dashboards.LifecycleState": { - "type": "string", - "enum": [ - "ACTIVE", - "TRASHED" + "oneOf": [ + { + "type": "string", + "enum": [ + "ACTIVE", + "TRASHED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "jobs.CleanRoomsNotebookTask": { @@ -3040,10 +3192,18 @@ ] }, "jobs.Condition": { - "type": "string", - "enum": [ - "ANY_UPDATED", - "ALL_UPDATED" + "oneOf": [ + { + "type": "string", + "enum": [ + "ANY_UPDATED", + "ALL_UPDATED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "jobs.ConditionTask": { @@ -3078,15 +3238,23 @@ ] }, "jobs.ConditionTaskOp": { - "type": "string", - "description": "* `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`.\n* `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” \u003e= “12”` will evaluate to `true`, `“10.0” \u003e= “12”` will evaluate to `false`.\n\nThe boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison.", - "enum": [ - "EQUAL_TO", - "GREATER_THAN", - "GREATER_THAN_OR_EQUAL", - "LESS_THAN", - "LESS_THAN_OR_EQUAL", - "NOT_EQUAL" + "oneOf": [ + { + "type": "string", + "description": "* `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`.\n* `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” \u003e= “12”` will evaluate to `true`, `“10.0” \u003e= “12”` will evaluate to `false`.\n\nThe boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison.", + "enum": [ + "EQUAL_TO", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL", + "LESS_THAN", + "LESS_THAN_OR_EQUAL", + "NOT_EQUAL" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "jobs.Continuous": { @@ -3242,23 +3410,39 @@ ] }, "jobs.Format": { - "type": "string", - "enum": [ - "SINGLE_TASK", - "MULTI_TASK" + "oneOf": [ + { + "type": "string", + "enum": [ + "SINGLE_TASK", + "MULTI_TASK" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "jobs.GitProvider": { - "type": "string", - "enum": [ - "gitHub", - "bitbucketCloud", - "azureDevOpsServices", - "gitHubEnterprise", - "bitbucketServer", - "gitLab", - "gitLabEnterpriseEdition", - "awsCodeCommit" + "oneOf": [ + { + "type": "string", + "enum": [ + "gitHub", + "bitbucketCloud", + "azureDevOpsServices", + "gitHubEnterprise", + "bitbucketServer", + "gitLab", + "gitLabEnterpriseEdition", + "awsCodeCommit" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "jobs.GitSnapshot": { @@ -3371,18 +3555,34 @@ ] }, "jobs.JobDeploymentKind": { - "type": "string", - "description": "* `BUNDLE`: The job is managed by Databricks Asset Bundle.", - "enum": [ - "BUNDLE" + "oneOf": [ + { + "type": "string", + "description": "* `BUNDLE`: The job is managed by Databricks Asset Bundle.", + "enum": [ + "BUNDLE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "jobs.JobEditMode": { - "type": "string", - "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified.", - "enum": [ - "UI_LOCKED", - "EDITABLE" + "oneOf": [ + { + "type": "string", + "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified.", + "enum": [ + "UI_LOCKED", + "EDITABLE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "jobs.JobEmailNotifications": { @@ -3550,29 +3750,53 @@ ] }, "jobs.JobSourceDirtyState": { - "type": "string", - "description": "Dirty state indicates the job is not fully synced with the job specification\nin the remote repository.\n\nPossible values are:\n* `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced.\n* `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced.", - "enum": [ - "NOT_SYNCED", - "DISCONNECTED" + "oneOf": [ + { + "type": "string", + "description": "Dirty state indicates the job is not fully synced with the job specification\nin the remote repository.\n\nPossible values are:\n* `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced.\n* `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced.", + "enum": [ + "NOT_SYNCED", + "DISCONNECTED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "jobs.JobsHealthMetric": { - "type": "string", - "description": "Specifies the health metric that is being evaluated for a particular health rule.\n\n* `RUN_DURATION_SECONDS`: Expected total time for a run in seconds.\n* `STREAMING_BACKLOG_BYTES`: An estimate of the maximum bytes of data waiting to be consumed across all streams. This metric is in Public Preview.\n* `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag across all streams. This metric is in Public Preview.\n* `STREAMING_BACKLOG_SECONDS`: An estimate of the maximum consumer delay across all streams. This metric is in Public Preview.\n* `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all streams. This metric is in Public Preview.", - "enum": [ - "RUN_DURATION_SECONDS", - "STREAMING_BACKLOG_BYTES", - "STREAMING_BACKLOG_RECORDS", - "STREAMING_BACKLOG_SECONDS", - "STREAMING_BACKLOG_FILES" + "oneOf": [ + { + "type": "string", + "description": "Specifies the health metric that is being evaluated for a particular health rule.\n\n* `RUN_DURATION_SECONDS`: Expected total time for a run in seconds.\n* `STREAMING_BACKLOG_BYTES`: An estimate of the maximum bytes of data waiting to be consumed across all streams. This metric is in Public Preview.\n* `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag across all streams. This metric is in Public Preview.\n* `STREAMING_BACKLOG_SECONDS`: An estimate of the maximum consumer delay across all streams. This metric is in Public Preview.\n* `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all streams. This metric is in Public Preview.", + "enum": [ + "RUN_DURATION_SECONDS", + "STREAMING_BACKLOG_BYTES", + "STREAMING_BACKLOG_RECORDS", + "STREAMING_BACKLOG_SECONDS", + "STREAMING_BACKLOG_FILES" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "jobs.JobsHealthOperator": { - "type": "string", - "description": "Specifies the operator used to compare the health metric value with the specified threshold.", - "enum": [ - "GREATER_THAN" + "oneOf": [ + { + "type": "string", + "description": "Specifies the operator used to compare the health metric value with the specified threshold.", + "enum": [ + "GREATER_THAN" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "jobs.JobsHealthRule": { @@ -3656,10 +3880,18 @@ ] }, "jobs.PauseStatus": { - "type": "string", - "enum": [ - "UNPAUSED", - "PAUSED" + "oneOf": [ + { + "type": "string", + "enum": [ + "UNPAUSED", + "PAUSED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "jobs.PeriodicTriggerConfiguration": { @@ -3689,11 +3921,19 @@ ] }, "jobs.PeriodicTriggerConfigurationTimeUnit": { - "type": "string", - "enum": [ - "HOURS", - "DAYS", - "WEEKS" + "oneOf": [ + { + "type": "string", + "enum": [ + "HOURS", + "DAYS", + "WEEKS" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "jobs.PipelineParams": { @@ -3795,15 +4035,23 @@ ] }, "jobs.RunIf": { - "type": "string", - "description": "An optional value indicating the condition that determines whether the task should be run once its dependencies have been completed. When omitted, defaults to `ALL_SUCCESS`.\n\nPossible values are:\n* `ALL_SUCCESS`: All dependencies have executed and succeeded\n* `AT_LEAST_ONE_SUCCESS`: At least one dependency has succeeded\n* `NONE_FAILED`: None of the dependencies have failed and at least one was executed\n* `ALL_DONE`: All dependencies have been completed\n* `AT_LEAST_ONE_FAILED`: At least one dependency failed\n* `ALL_FAILED`: ALl dependencies have failed", - "enum": [ - "ALL_SUCCESS", - "ALL_DONE", - "NONE_FAILED", - "AT_LEAST_ONE_SUCCESS", - "ALL_FAILED", - "AT_LEAST_ONE_FAILED" + "oneOf": [ + { + "type": "string", + "description": "An optional value indicating the condition that determines whether the task should be run once its dependencies have been completed. When omitted, defaults to `ALL_SUCCESS`.\n\nPossible values are:\n* `ALL_SUCCESS`: All dependencies have executed and succeeded\n* `AT_LEAST_ONE_SUCCESS`: At least one dependency has succeeded\n* `NONE_FAILED`: None of the dependencies have failed and at least one was executed\n* `ALL_DONE`: All dependencies have been completed\n* `AT_LEAST_ONE_FAILED`: At least one dependency failed\n* `ALL_FAILED`: ALl dependencies have failed", + "enum": [ + "ALL_SUCCESS", + "ALL_DONE", + "NONE_FAILED", + "AT_LEAST_ONE_SUCCESS", + "ALL_FAILED", + "AT_LEAST_ONE_FAILED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "jobs.RunJobTask": { @@ -3863,11 +4111,19 @@ ] }, "jobs.Source": { - "type": "string", - "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\\\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider.", - "enum": [ - "WORKSPACE", - "GIT" + "oneOf": [ + { + "type": "string", + "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\\\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider.", + "enum": [ + "WORKSPACE", + "GIT" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "jobs.SparkJarTask": { @@ -4567,12 +4823,20 @@ ] }, "ml.ModelVersionStatus": { - "type": "string", - "description": "Current status of `model_version`", - "enum": [ - "PENDING_REGISTRATION", - "FAILED_REGISTRATION", - "READY" + "oneOf": [ + { + "type": "string", + "description": "Current status of `model_version`", + "enum": [ + "PENDING_REGISTRATION", + "FAILED_REGISTRATION", + "READY" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "ml.ModelVersionTag": { @@ -4618,23 +4882,39 @@ ] }, "pipelines.DayOfWeek": { - "type": "string", - "description": "Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour).\nIf not specified all days of the week will be used.", - "enum": [ - "MONDAY", - "TUESDAY", - "WEDNESDAY", - "THURSDAY", - "FRIDAY", - "SATURDAY", - "SUNDAY" + "oneOf": [ + { + "type": "string", + "description": "Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour).\nIf not specified all days of the week will be used.", + "enum": [ + "MONDAY", + "TUESDAY", + "WEDNESDAY", + "THURSDAY", + "FRIDAY", + "SATURDAY", + "SUNDAY" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "pipelines.DeploymentKind": { - "type": "string", - "description": "The deployment method that manages the pipeline:\n- BUNDLE: The pipeline is managed by a Databricks Asset Bundle.\n", - "enum": [ - "BUNDLE" + "oneOf": [ + { + "type": "string", + "description": "The deployment method that manages the pipeline:\n- BUNDLE: The pipeline is managed by a Databricks Asset Bundle.\n", + "enum": [ + "BUNDLE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "pipelines.FileLibrary": { @@ -4940,11 +5220,19 @@ ] }, "pipelines.PipelineClusterAutoscaleMode": { - "type": "string", - "description": "Databricks Enhanced Autoscaling optimizes cluster utilization by automatically\nallocating cluster resources based on workload volume, with minimal impact to\nthe data processing latency of your pipelines. Enhanced Autoscaling is available\nfor `updates` clusters only. The legacy autoscaling feature is used for `maintenance`\nclusters.\n", - "enum": [ - "ENHANCED", - "LEGACY" + "oneOf": [ + { + "type": "string", + "description": "Databricks Enhanced Autoscaling optimizes cluster utilization by automatically\nallocating cluster resources based on workload volume, with minimal impact to\nthe data processing latency of your pipelines. Enhanced Autoscaling is available\nfor `updates` clusters only. The legacy autoscaling feature is used for `maintenance`\nclusters.\n", + "enum": [ + "ENHANCED", + "LEGACY" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "pipelines.PipelineDeployment": { @@ -5193,11 +5481,19 @@ ] }, "pipelines.TableSpecificConfigScdType": { - "type": "string", - "description": "The SCD type to use to ingest the table.", - "enum": [ - "SCD_TYPE_1", - "SCD_TYPE_2" + "oneOf": [ + { + "type": "string", + "description": "The SCD type to use to ingest the table.", + "enum": [ + "SCD_TYPE_1", + "SCD_TYPE_2" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "serving.Ai21LabsConfig": { @@ -5304,11 +5600,19 @@ ] }, "serving.AiGatewayGuardrailPiiBehaviorBehavior": { - "type": "string", - "description": "Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.", - "enum": [ - "NONE", - "BLOCK" + "oneOf": [ + { + "type": "string", + "description": "Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.", + "enum": [ + "NONE", + "BLOCK" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "serving.AiGatewayGuardrails": { @@ -5394,18 +5698,34 @@ ] }, "serving.AiGatewayRateLimitKey": { - "type": "string", - "description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.", - "enum": [ - "user", - "endpoint" + "oneOf": [ + { + "type": "string", + "description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.", + "enum": [ + "user", + "endpoint" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "serving.AiGatewayRateLimitRenewalPeriod": { - "type": "string", - "description": "Renewal period field for a rate limit. Currently, only 'minute' is supported.", - "enum": [ - "minute" + "oneOf": [ + { + "type": "string", + "description": "Renewal period field for a rate limit. Currently, only 'minute' is supported.", + "enum": [ + "minute" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "serving.AiGatewayUsageTrackingConfig": { @@ -5469,13 +5789,21 @@ ] }, "serving.AmazonBedrockConfigBedrockProvider": { - "type": "string", - "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.", - "enum": [ - "anthropic", - "cohere", - "ai21labs", - "amazon" + "oneOf": [ + { + "type": "string", + "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.", + "enum": [ + "anthropic", + "cohere", + "ai21labs", + "amazon" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "serving.AnthropicConfig": { @@ -5704,17 +6032,25 @@ ] }, "serving.ExternalModelProvider": { - "type": "string", - "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n", - "enum": [ - "ai21labs", - "anthropic", - "amazon-bedrock", - "cohere", - "databricks-model-serving", - "google-cloud-vertex-ai", - "openai", - "palm" + "oneOf": [ + { + "type": "string", + "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n", + "enum": [ + "ai21labs", + "anthropic", + "amazon-bedrock", + "cohere", + "databricks-model-serving", + "google-cloud-vertex-ai", + "openai", + "palm" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "serving.GoogleCloudVertexAiConfig": { @@ -5858,18 +6194,34 @@ ] }, "serving.RateLimitKey": { - "type": "string", - "description": "Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.", - "enum": [ - "user", - "endpoint" + "oneOf": [ + { + "type": "string", + "description": "Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.", + "enum": [ + "user", + "endpoint" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "serving.RateLimitRenewalPeriod": { - "type": "string", - "description": "Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported.", - "enum": [ - "minute" + "oneOf": [ + { + "type": "string", + "description": "Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported.", + "enum": [ + "minute" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "serving.Route": { @@ -6016,23 +6368,39 @@ ] }, "serving.ServedModelInputWorkloadSize": { - "type": "string", - "description": "The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.\n", - "enum": [ - "Small", - "Medium", - "Large" + "oneOf": [ + { + "type": "string", + "description": "The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.\n", + "enum": [ + "Small", + "Medium", + "Large" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "serving.ServedModelInputWorkloadType": { - "type": "string", - "description": "The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n", - "enum": [ - "CPU", - "GPU_SMALL", - "GPU_MEDIUM", - "GPU_LARGE", - "MULTIGPU_MEDIUM" + "oneOf": [ + { + "type": "string", + "description": "The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n", + "enum": [ + "CPU", + "GPU_SMALL", + "GPU_MEDIUM", + "GPU_LARGE", + "MULTIGPU_MEDIUM" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } ] }, "serving.TrafficConfig": { From e9902036b8149df9f72c6358a0450178b8ba965d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 22 Jan 2025 11:50:13 +0100 Subject: [PATCH 4/6] Set WorktreeRoot to sync root outside git repo (#2197) ## Changes If git is not detected, set default worktree root to sync root. Otherwise NewFileSet/View raise an error about worktree root being outside view root in acceptance/bundle/sync-paths-dotdot. This behavior is introduced in https://github.com/databricks/cli/pull/1945 Stacked on https://github.com/databricks/cli/pull/2202 ## Tests Existing tests. --- acceptance/bundle/syncroot/dotdot-nogit/output.txt | 6 +----- bundle/config/mutator/load_git_details.go | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/syncroot/dotdot-nogit/output.txt b/acceptance/bundle/syncroot/dotdot-nogit/output.txt index 34059e276..46f617f35 100644 --- a/acceptance/bundle/syncroot/dotdot-nogit/output.txt +++ b/acceptance/bundle/syncroot/dotdot-nogit/output.txt @@ -1,11 +1,7 @@ -Error: path "$TMPDIR_PARENT" is not within repository root "$TMPDIR" - Name: test-bundle Target: default Workspace: User: $USERNAME Path: /Workspace/Users/$USERNAME/.bundle/test-bundle/default -Found 1 error - -Exit code: 1 +Validation OK! diff --git a/bundle/config/mutator/load_git_details.go b/bundle/config/mutator/load_git_details.go index 5c263ac03..3661c6bcd 100644 --- a/bundle/config/mutator/load_git_details.go +++ b/bundle/config/mutator/load_git_details.go @@ -32,7 +32,7 @@ func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn } if info.WorktreeRoot == "" { - b.WorktreeRoot = b.BundleRoot + b.WorktreeRoot = b.SyncRoot } else { b.WorktreeRoot = vfs.MustNew(info.WorktreeRoot) } From 876526a19a8f65e5ecc8fd22779dcefef8115c4a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 22 Jan 2025 12:20:49 +0100 Subject: [PATCH 5/6] Use local git config in tests (#2205) I've seen this error: could not lock config file $TMPDIR_GPARENT/TestAccept3968313522/002/.gitconfig: File exists This is likely the cause. --- acceptance/script.prepare | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/script.prepare b/acceptance/script.prepare index 61061b59e..5900016d7 100644 --- a/acceptance/script.prepare +++ b/acceptance/script.prepare @@ -34,7 +34,7 @@ trace() { git-repo-init() { git init -qb main - git config --global core.autocrlf false + git config core.autocrlf false git config user.name "Tester" git config user.email "tester@databricks.com" git add databricks.yml From 6c3ddbd921c568c3937d52a7167ae63fa3cbf533 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 22 Jan 2025 17:44:54 +0530 Subject: [PATCH 6/6] Add `auth.Env` function (#2204) ## Changes `auth.Env` is a generic function that we can use for authenticated tools downstream to the CLI. ## Tests Unit test. --- bundle/bundle.go | 20 ++------------------ libs/auth/env.go | 26 ++++++++++++++++++++++++++ libs/auth/env_test.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 18 deletions(-) create mode 100644 libs/auth/env.go create mode 100644 libs/auth/env_test.go diff --git a/bundle/bundle.go b/bundle/bundle.go index 3bf4ffb62..e715b8b2c 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -17,6 +17,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/bundle/metadata" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/fileset" "github.com/databricks/cli/libs/locker" "github.com/databricks/cli/libs/log" @@ -24,7 +25,6 @@ import ( "github.com/databricks/cli/libs/terraform" "github.com/databricks/cli/libs/vfs" "github.com/databricks/databricks-sdk-go" - sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/hashicorp/terraform-exec/tfexec" ) @@ -242,21 +242,5 @@ func (b *Bundle) AuthEnv() (map[string]string, error) { } cfg := b.client.Config - out := make(map[string]string) - for _, attr := range sdkconfig.ConfigAttributes { - // Ignore profile so that downstream tools don't try and reload - // the profile even though we know the current configuration is valid. - if attr.Name == "profile" { - continue - } - if len(attr.EnvVars) == 0 { - continue - } - if attr.IsZero(cfg) { - continue - } - out[attr.EnvVars[0]] = attr.GetString(cfg) - } - - return out, nil + return auth.Env(cfg), nil } diff --git a/libs/auth/env.go b/libs/auth/env.go new file mode 100644 index 000000000..c58cc53e3 --- /dev/null +++ b/libs/auth/env.go @@ -0,0 +1,26 @@ +package auth + +import "github.com/databricks/databricks-sdk-go/config" + +// Env generates the authentication environment variables we need to set for +// downstream applications from the CLI to work correctly. +func Env(cfg *config.Config) map[string]string { + out := make(map[string]string) + for _, attr := range config.ConfigAttributes { + // Ignore profile so that downstream tools don't try and reload + // the profile. We know the current configuration is already valid since + // otherwise the CLI would have thrown an error when loading it. + if attr.Name == "profile" { + continue + } + if len(attr.EnvVars) == 0 { + continue + } + if attr.IsZero(cfg) { + continue + } + out[attr.EnvVars[0]] = attr.GetString(cfg) + } + + return out +} diff --git a/libs/auth/env_test.go b/libs/auth/env_test.go new file mode 100644 index 000000000..be1cfc7ac --- /dev/null +++ b/libs/auth/env_test.go @@ -0,0 +1,42 @@ +package auth + +import ( + "testing" + + "github.com/databricks/databricks-sdk-go/config" + "github.com/stretchr/testify/assert" +) + +func TestAuthEnv(t *testing.T) { + in := &config.Config{ + Profile: "thisshouldbeignored", + Host: "https://test.com", + Token: "test-token", + Password: "test-password", + MetadataServiceURL: "http://somurl.com", + + AzureUseMSI: true, + AzureTenantID: "test-tenant-id", + AzureClientID: "test-client-id", + AzureClientSecret: "test-client-secret", + + ActionsIDTokenRequestToken: "test-actions-id-token-request-token", + } + + expected := map[string]string{ + "DATABRICKS_HOST": "https://test.com", + "DATABRICKS_TOKEN": "test-token", + "DATABRICKS_PASSWORD": "test-password", + "DATABRICKS_METADATA_SERVICE_URL": "http://somurl.com", + + "ARM_USE_MSI": "true", + "ARM_TENANT_ID": "test-tenant-id", + "ARM_CLIENT_ID": "test-client-id", + "ARM_CLIENT_SECRET": "test-client-secret", + + "ACTIONS_ID_TOKEN_REQUEST_TOKEN": "test-actions-id-token-request-token", + } + + out := Env(in) + assert.Equal(t, expected, out) +}